Compare commits
16 Commits
bc2ade38d9
...
v0.12.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 |
276
docs/cli.md
276
docs/cli.md
@@ -6,7 +6,9 @@
|
|||||||
narratio run --session-id 2026-04-04
|
narratio run --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
This command uses default discovery for `pipeline.yml` and `session.yml`; both files must be discoverable unless you pass explicit `--config` and `--session` paths.
|
This command uses default system discovery for `pipeline.yml`, `campaign.yml`, and local `session.yml`. If local session discovery misses and S3 storage is configured, `--session-id` can load remote `session.yml` from the canonical session prefix.
|
||||||
|
|
||||||
|
Default discovery checks system config locations only. Pass `--config`, `--campaign`, and `--session` to use files from the current working directory.
|
||||||
|
|
||||||
## Command Overview
|
## Command Overview
|
||||||
|
|
||||||
@@ -15,9 +17,15 @@ Implemented commands:
|
|||||||
- `run`: execute pipeline stages and persist manifest state.
|
- `run`: execute pipeline stages and persist manifest state.
|
||||||
- `plan`: validate config, prepare workspace layout, and print stage run/skip decisions.
|
- `plan`: validate config, prepare workspace layout, and print stage run/skip decisions.
|
||||||
- `resume`: continue from first non-succeeded stage unless forced.
|
- `resume`: continue from first non-succeeded stage unless forced.
|
||||||
- `status`: read and print stage statuses from an existing manifest.
|
- `status`: read an existing manifest or inspect local/remote state for a session.
|
||||||
- `run-stage`: execute exactly one stage.
|
- `run-stage`: execute exactly one stage.
|
||||||
|
- `analyze`: force-rerun the analyze stage.
|
||||||
- `restore`: restore durable local session state from the committed remote archive state.
|
- `restore`: restore durable local session state from the committed remote archive state.
|
||||||
|
- `session validate`: run read-only preflight checks for a session.
|
||||||
|
- `session init`: create local or remote `session.yml`.
|
||||||
|
- `artifacts list`: list effective artifact source IDs.
|
||||||
|
- `locks`: list, add, and remove archive promotion locks.
|
||||||
|
- `clean`: remove local workspace/spool state for one session or all local sessions.
|
||||||
|
|
||||||
Unknown commands print usage and exit non-zero.
|
Unknown commands print usage and exit non-zero.
|
||||||
|
|
||||||
@@ -28,6 +36,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
### `run`
|
### `run`
|
||||||
|
|
||||||
- `--config <path>`: optional explicit `pipeline.yml` path.
|
- `--config <path>`: optional explicit `pipeline.yml` path.
|
||||||
|
- `--campaign <path>`: optional explicit `campaign.yml` path.
|
||||||
- `--session <path>`: optional explicit `session.yml` path.
|
- `--session <path>`: optional explicit `session.yml` path.
|
||||||
- `--session-id <value>`: session template variable value.
|
- `--session-id <value>`: session template variable value.
|
||||||
- `--previous-session-id <value>`: previous-session template variable value.
|
- `--previous-session-id <value>`: previous-session template variable value.
|
||||||
@@ -37,6 +46,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
### `plan`
|
### `plan`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
- `--previous-session-id <value>`
|
- `--previous-session-id <value>`
|
||||||
@@ -45,6 +55,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
### `resume`
|
### `resume`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
- `--previous-session-id <value>`
|
- `--previous-session-id <value>`
|
||||||
@@ -54,6 +65,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
### `run-stage`
|
### `run-stage`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
- `--previous-session-id <value>`
|
- `--previous-session-id <value>`
|
||||||
@@ -61,6 +73,17 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||||
- positional `<stage>`: required stage name.
|
- positional `<stage>`: required stage name.
|
||||||
|
|
||||||
|
### `analyze`
|
||||||
|
|
||||||
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
|
- `--session <path>`
|
||||||
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||||
|
|
||||||
|
`analyze` is force-by-design and does not accept `--force`.
|
||||||
|
|
||||||
Valid stage names:
|
Valid stage names:
|
||||||
|
|
||||||
- `prepare`
|
- `prepare`
|
||||||
@@ -76,6 +99,7 @@ Valid stage names:
|
|||||||
### `restore`
|
### `restore`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
- `--previous-session-id <value>`
|
- `--previous-session-id <value>`
|
||||||
@@ -83,9 +107,68 @@ Valid stage names:
|
|||||||
- `--force`: overwrite local conflicting files with remote archive files.
|
- `--force`: overwrite local conflicting files with remote archive files.
|
||||||
- `--include-audio`: include durable archived `audio/**` files in restore scope.
|
- `--include-audio`: include durable archived `audio/**` files in restore scope.
|
||||||
|
|
||||||
|
### `clean`
|
||||||
|
|
||||||
|
- `--session-id <value>`: required for session cleanup unless `--all` is set.
|
||||||
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
|
- `--session <path>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
- `--all`: clean all local session work/spool state using pipeline config only.
|
||||||
|
- `--dry-run`: print cleanup targets without deleting.
|
||||||
|
- `--clear-cache`: also remove matching S3 audio cache entries.
|
||||||
|
|
||||||
### `status`
|
### `status`
|
||||||
|
|
||||||
- `--manifest <path>`: required manifest path.
|
- `--manifest <path>`: inspect one manifest file.
|
||||||
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
|
- `--session <path>`
|
||||||
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
|
||||||
|
### `session validate`
|
||||||
|
|
||||||
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
|
- `--session <path>`
|
||||||
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
|
||||||
|
### `session init`
|
||||||
|
|
||||||
|
- `--config <path>`: required.
|
||||||
|
- `--campaign <path>`: required.
|
||||||
|
- `--session-id <value>`: required.
|
||||||
|
- `--output <path>`: local `session.yml` target; mutually exclusive with `--remote`.
|
||||||
|
- `--remote`: write remote `session.yml` to the canonical session prefix; mutually exclusive with `--output`.
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
- `--date <value>`
|
||||||
|
- `--title <value>`
|
||||||
|
- `--audio-s3-prefix <prefix>`: defaults to `audio/` when neither audio flag is provided.
|
||||||
|
- `--audio-dir <path>`: local audio directory; mutually exclusive with `--audio-s3-prefix`.
|
||||||
|
- `--force`: overwrite existing local or remote target.
|
||||||
|
|
||||||
|
### `artifacts list`
|
||||||
|
|
||||||
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
|
- `--session <path>`
|
||||||
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
|
- `--remote`: check remote availability for configured archive promotion destinations.
|
||||||
|
|
||||||
|
### `locks`
|
||||||
|
|
||||||
|
- `--session-id <value>`: required for list, add, and remove.
|
||||||
|
- `--config <path>`: optional explicit `pipeline.yml` path.
|
||||||
|
- `--campaign <path>`: optional explicit `campaign.yml` path.
|
||||||
|
- `--session <path>`: optional explicit `session.yml` path.
|
||||||
|
- `--previous-session-id <value>`: optional session template value.
|
||||||
|
- `add <source>`: add a remote lock for one artifact or transcript source.
|
||||||
|
- `add --reason <text>`: record an optional remote lock reason.
|
||||||
|
- `add --force`: update the reason for an existing remote lock.
|
||||||
|
- `remove <source>`: remove one remote lock.
|
||||||
|
|
||||||
## Command Reference
|
## Command Reference
|
||||||
|
|
||||||
@@ -97,14 +180,15 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio run [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||||
|
|
||||||
Common failure cases:
|
Common failure cases:
|
||||||
- missing default config/session paths when flags omitted.
|
- missing system default config/campaign/session paths when flags omitted.
|
||||||
|
- missing local session plus missing/unavailable remote `session.yml`.
|
||||||
- invalid template/rendered session mismatch.
|
- invalid template/rendered session mismatch.
|
||||||
- unknown/invalid `--artifacts` value.
|
- unknown/invalid `--artifacts` value.
|
||||||
- `--artifacts` with unknown configured artifact key.
|
- `--artifacts` with unknown configured artifact key.
|
||||||
@@ -117,7 +201,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio plan [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
|
narratio plan [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output includes:
|
Success output includes:
|
||||||
@@ -126,7 +210,8 @@ Success output includes:
|
|||||||
- `totals: run=<n> skip=<n>`
|
- `totals: run=<n> skip=<n>`
|
||||||
|
|
||||||
Common failure cases:
|
Common failure cases:
|
||||||
- same config/session discovery and validation failures as `run`.
|
- same config/campaign/session discovery and validation failures as `run`.
|
||||||
|
- remote session fallback failures when local session discovery misses.
|
||||||
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
|
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
|
||||||
|
|
||||||
### `resume`
|
### `resume`
|
||||||
@@ -137,7 +222,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio resume [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio resume [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
@@ -152,22 +237,111 @@ Common failure cases:
|
|||||||
### `status`
|
### `status`
|
||||||
|
|
||||||
Purpose:
|
Purpose:
|
||||||
- Inspect one manifest file without executing stages.
|
- Inspect one manifest file, or inspect configured local/remote state for a session.
|
||||||
|
|
||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio status --manifest <manifest.json>
|
narratio status --manifest <manifest.json>
|
||||||
|
narratio status [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output includes:
|
Manifest output includes:
|
||||||
- `session_id: <id>`
|
- `session_id: <id>`
|
||||||
- `updated_at: <timestamp>`
|
- `updated_at: <timestamp>`
|
||||||
- `stages:` entries (`- <stage>: <status>`)
|
- `stages:` entries (`- <stage>: <status>`)
|
||||||
|
|
||||||
|
Session output includes:
|
||||||
|
- session ID, campaign, workspace, session config source.
|
||||||
|
- local manifest state when present.
|
||||||
|
- remote current archive state when storage is configured.
|
||||||
|
- catalog-based remote output availability for expected transcript and artifact sources.
|
||||||
|
- effective archive locks and conservative next actions.
|
||||||
|
|
||||||
Common failure cases:
|
Common failure cases:
|
||||||
- missing `--manifest`.
|
- missing `--manifest` when no config/session flags are provided.
|
||||||
- unreadable or invalid manifest path.
|
- unreadable or invalid manifest path.
|
||||||
|
- invalid config or remote session fallback failure in session mode.
|
||||||
|
|
||||||
|
### `session validate`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- Run read-only preflight checks for a session.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
|
||||||
|
```
|
||||||
|
|
||||||
|
Checks include:
|
||||||
|
- effective config and session source.
|
||||||
|
- stable input files.
|
||||||
|
- local or remote audio availability.
|
||||||
|
- previous-session requirements.
|
||||||
|
- archive promotions and effective locks.
|
||||||
|
|
||||||
|
Warnings do not fail the command. Any `ERROR` finding exits non-zero.
|
||||||
|
|
||||||
|
### `session init`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- Create a strict-decoded session skeleton locally or in object storage.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --output ./session.yml
|
||||||
|
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --remote
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- exactly one of `--output` or `--remote` is required.
|
||||||
|
- remote writes target `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
||||||
|
- existing local or remote targets fail unless `--force` is passed.
|
||||||
|
- remote writes use existence checks, not compare-and-swap.
|
||||||
|
|
||||||
|
### `artifacts list`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- List built-in, configured, previous-session, promoted, and locked artifact sources.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio artifacts list [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--remote]
|
||||||
|
```
|
||||||
|
|
||||||
|
`--remote` checks promoted top-level object availability through the storage adapter. Remote markers appear only in the `Promoted` section, which reports each configured archive promotion destination and includes `dest=<path>` when that destination differs from the source's canonical path.
|
||||||
|
|
||||||
|
### `locks`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- Inspect and mutate source-based archive promotion locks for one session.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio locks --session-id <id>
|
||||||
|
narratio locks add --session-id <id> [--reason <text>] [--force] <source>
|
||||||
|
narratio locks remove --session-id <id> <source>
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- `--session-id` is required for list, add, and remove.
|
||||||
|
- optional `--config`, `--campaign`, and `--session` override default config discovery.
|
||||||
|
- list mode prints effective locks from static `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`.
|
||||||
|
- `locks add` writes only the remote lock store and fails if the source is already locked by pipeline config.
|
||||||
|
- `locks remove` removes only remote locks and cannot remove static pipeline locks.
|
||||||
|
- `locks add --force` is required to update an existing remote lock reason.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio locks --session-id 2026-04-04
|
||||||
|
narratio locks add --session-id 2026-04-04 --reason "manual transcript review" narratio.transcript.trimmed
|
||||||
|
narratio locks remove --session-id 2026-04-04 narratio.transcript.trimmed
|
||||||
|
```
|
||||||
|
|
||||||
### `run-stage`
|
### `run-stage`
|
||||||
|
|
||||||
@@ -177,7 +351,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
narratio run-stage [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
@@ -193,6 +367,26 @@ Common failure cases:
|
|||||||
- unknown stage name.
|
- unknown stage name.
|
||||||
- using `--artifacts` with any non-`analyze` stage.
|
- using `--artifacts` with any non-`analyze` stage.
|
||||||
|
|
||||||
|
### `analyze`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- Force-rerun the analyze stage.
|
||||||
|
- Provide a shorter equivalent for `narratio run-stage --force analyze`.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio analyze [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
|
||||||
|
```
|
||||||
|
|
||||||
|
Success output:
|
||||||
|
- `narratio analyze: executed=<n> skipped=<n> force=true; manifest=<path>`
|
||||||
|
|
||||||
|
Common failure cases:
|
||||||
|
- positional arguments.
|
||||||
|
- `--force`, because force is implicit.
|
||||||
|
- unknown configured artifact keys.
|
||||||
|
|
||||||
### `restore`
|
### `restore`
|
||||||
|
|
||||||
Purpose:
|
Purpose:
|
||||||
@@ -201,7 +395,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
narratio restore [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output (dry-run):
|
Success output (dry-run):
|
||||||
@@ -226,6 +420,40 @@ Common failure cases:
|
|||||||
- local conflicts without `--force`.
|
- local conflicts without `--force`.
|
||||||
- session lock conflict.
|
- session lock conflict.
|
||||||
|
|
||||||
|
When `--include-audio` is set, S3 audio files are restored through the shared audio cache. Cache hits avoid re-downloading large audio objects.
|
||||||
|
|
||||||
|
### `clean`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
- Remove local Narratio work/spool state for testing, reruns, or recovery from corrupted local files.
|
||||||
|
- Preserve durable S3 audio cache state unless `--clear-cache` is passed.
|
||||||
|
|
||||||
|
Syntax:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean --session-id <id> [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--clear-cache]
|
||||||
|
narratio clean --all [--config <pipeline.yml>] [--dry-run] [--clear-cache]
|
||||||
|
```
|
||||||
|
|
||||||
|
Session cleanup deletes:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||||
|
- `{spool.root}/{campaign}/{session_id}`
|
||||||
|
|
||||||
|
All-session cleanup deletes:
|
||||||
|
- `{workspace.root}/work`
|
||||||
|
- the contents of `{spool.root}`, while preserving the spool root directory itself.
|
||||||
|
|
||||||
|
Cache behavior:
|
||||||
|
- cache is preserved by default.
|
||||||
|
- `--clear-cache` in session mode removes cached S3 audio files for the resolved session.
|
||||||
|
- `--all --clear-cache` removes the configured Narratio S3 audio cache namespace for the configured bucket/root prefix.
|
||||||
|
- `--clear-cache` does not delete arbitrary files under `pipeline.cache.root`.
|
||||||
|
|
||||||
|
Common failure cases:
|
||||||
|
- missing `--session-id` when `--all` is not set.
|
||||||
|
- combining `--all` with `--campaign`, `--session`, `--session-id`, or `--previous-session-id`.
|
||||||
|
- unsafe cleanup target, such as a symlink, a non-directory session target, a configured root directory, or a path outside the configured root.
|
||||||
|
|
||||||
## Common Workflows
|
## Common Workflows
|
||||||
|
|
||||||
Default-discovery run:
|
Default-discovery run:
|
||||||
@@ -246,10 +474,10 @@ Resume with selected analyze artifacts:
|
|||||||
narratio resume --session-id 2026-04-04 --artifacts player_handout
|
narratio resume --session-id 2026-04-04 --artifacts player_handout
|
||||||
```
|
```
|
||||||
|
|
||||||
Run only analyze stage with selected artifacts:
|
Force-rerun analyze with selected artifacts:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage --session-id 2026-04-04 --artifacts player_handout analyze
|
narratio analyze --session-id 2026-04-04 --artifacts player_handout
|
||||||
```
|
```
|
||||||
|
|
||||||
Preview restore actions without writes:
|
Preview restore actions without writes:
|
||||||
@@ -262,7 +490,7 @@ Restore and then force analyze:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore --session-id 2026-04-04
|
narratio restore --session-id 2026-04-04
|
||||||
narratio run-stage --session-id 2026-04-04 --force analyze
|
narratio analyze --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Rehydrate canonical previous-session inputs after artifact-input changes:
|
Rehydrate canonical previous-session inputs after artifact-input changes:
|
||||||
@@ -271,6 +499,20 @@ Rehydrate canonical previous-session inputs after artifact-input changes:
|
|||||||
narratio run-stage --session-id 2026-04-04 --force prepare
|
narratio run-stage --session-id 2026-04-04 --force prepare
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Reset local state before testing restore:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean --session-id 2026-04-04 --dry-run
|
||||||
|
narratio clean --session-id 2026-04-04
|
||||||
|
narratio restore --session-id 2026-04-04 --include-audio
|
||||||
|
```
|
||||||
|
|
||||||
|
Clean all local sessions while keeping cached S3 audio:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean --all
|
||||||
|
```
|
||||||
|
|
||||||
## Diagnostic / Recovery Commands
|
## Diagnostic / Recovery Commands
|
||||||
|
|
||||||
Inspect stage status:
|
Inspect stage status:
|
||||||
@@ -280,7 +522,7 @@ narratio status --manifest <manifest.json>
|
|||||||
```
|
```
|
||||||
|
|
||||||
Get manifest path from previous output:
|
Get manifest path from previous output:
|
||||||
- `run`, `resume`, and `run-stage` print `manifest=<path>` on success.
|
- `run`, `resume`, `run-stage`, and `analyze` print `manifest=<path>` on success.
|
||||||
|
|
||||||
## `--artifacts` and `--force`
|
## `--artifacts` and `--force`
|
||||||
|
|
||||||
|
|||||||
190
docs/config.md
190
docs/config.md
@@ -2,12 +2,13 @@
|
|||||||
|
|
||||||
## 1. Overview
|
## 1. Overview
|
||||||
|
|
||||||
Narratio loads two YAML files:
|
Narratio loads three YAML files:
|
||||||
|
|
||||||
- `pipeline.yml`: pipeline-level runtime configuration.
|
- `pipeline.yml`: pipeline-level runtime configuration.
|
||||||
- `session.yml`: per-session metadata and input selection.
|
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
|
||||||
|
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
|
||||||
|
|
||||||
These commands load and validate both files before running:
|
These commands load and validate all three files before running:
|
||||||
|
|
||||||
- `narratio run`
|
- `narratio run`
|
||||||
- `narratio plan`
|
- `narratio plan`
|
||||||
@@ -19,12 +20,23 @@ Behavior:
|
|||||||
|
|
||||||
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
||||||
- session templates render before session YAML decode.
|
- session templates render before session YAML decode.
|
||||||
|
- remote `session.yml` uses the same strict decode and template behavior as local `session.yml`.
|
||||||
- defaults are applied for optional pipeline fields.
|
- defaults are applied for optional pipeline fields.
|
||||||
|
- campaign-level stable input paths fill missing session input paths.
|
||||||
|
- session-level stable input paths override campaign-level input paths.
|
||||||
- validation enforces required fields, value formats, and cross-field constraints.
|
- validation enforces required fields, value formats, and cross-field constraints.
|
||||||
|
|
||||||
## 2. Config file discovery
|
## 2. Config file discovery
|
||||||
|
|
||||||
Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
These commands use the same config discovery behavior:
|
||||||
|
|
||||||
|
- `narratio run`
|
||||||
|
- `narratio plan`
|
||||||
|
- `narratio resume`
|
||||||
|
- `narratio run-stage`
|
||||||
|
- `narratio restore`
|
||||||
|
|
||||||
|
Pipeline config lookup:
|
||||||
|
|
||||||
- if `--config <path>` is provided, that path is used.
|
- if `--config <path>` is provided, that path is used.
|
||||||
- if omitted, Narratio searches in order:
|
- if omitted, Narratio searches in order:
|
||||||
@@ -32,18 +44,29 @@ Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
|||||||
2. `/etc/narratio/pipeline.yml`
|
2. `/etc/narratio/pipeline.yml`
|
||||||
- first existing file wins.
|
- first existing file wins.
|
||||||
|
|
||||||
## 3. Session file discovery and templating
|
Campaign config lookup:
|
||||||
|
|
||||||
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
- if `--campaign <path>` is provided, that path is used.
|
||||||
|
|
||||||
- if `--session <path>` is provided, that path is used.
|
|
||||||
- if omitted, Narratio searches in order:
|
- if omitted, Narratio searches in order:
|
||||||
1. `./session.yml`
|
1. `/usr/local/etc/narratio/campaign.yml`
|
||||||
2. `/usr/local/etc/narratio/session.yml`
|
2. `/etc/narratio/campaign.yml`
|
||||||
3. `/etc/narratio/session.yml`
|
|
||||||
- first existing file wins.
|
- first existing file wins.
|
||||||
|
|
||||||
Template behavior:
|
Session config lookup:
|
||||||
|
|
||||||
|
- if `--session <path>` is provided, that path is used.
|
||||||
|
- if `--session` is omitted, Narratio searches locally in order:
|
||||||
|
1. `/usr/local/etc/narratio/session.yml`
|
||||||
|
2. `/etc/narratio/session.yml`
|
||||||
|
- first existing local file wins.
|
||||||
|
- if no local session file is found, `--session-id <value>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
|
||||||
|
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||||
|
- local discovery always runs before remote fallback.
|
||||||
|
- local files in the current working directory are used only when passed explicitly, for example `--config ./pipeline.yml --campaign ./campaign.yml --session ./session.yml`.
|
||||||
|
|
||||||
|
## 3. Session templating
|
||||||
|
|
||||||
|
Template behavior for local and remote `session.yml`:
|
||||||
|
|
||||||
- supported placeholders:
|
- supported placeholders:
|
||||||
- `{{session_id}}`
|
- `{{session_id}}`
|
||||||
@@ -56,7 +79,9 @@ Template behavior:
|
|||||||
- if rendered `session_id` mismatches `--session-id`, load fails.
|
- if rendered `session_id` mismatches `--session-id`, load fails.
|
||||||
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
|
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
|
||||||
|
|
||||||
## 4. Minimal pipeline config
|
## 4. Minimal config set
|
||||||
|
|
||||||
|
### `pipeline.yml`
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -69,22 +94,40 @@ Why this is sufficient:
|
|||||||
- `workspace.root` defaults to `/var/lib/narratio`.
|
- `workspace.root` defaults to `/var/lib/narratio`.
|
||||||
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
|
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
|
||||||
|
|
||||||
## 5. Minimal session template
|
### `campaign.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Why this is sufficient:
|
||||||
|
|
||||||
|
- `campaign` supplies the stable campaign identity.
|
||||||
|
- stable input files are required and resolve relative to `campaign.yml` when copied during `prepare`.
|
||||||
|
|
||||||
|
### `session.yml`
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
session_id: "{{ session_id }}"
|
session_id: "{{ session_id }}"
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Usage:
|
Why this is sufficient:
|
||||||
|
|
||||||
|
- `session_id` is required and can be rendered from `--session-id`.
|
||||||
|
- `campaign` can be omitted because it is supplied by `campaign.yml`.
|
||||||
|
- stable input paths can be omitted because `campaign.yml` supplies defaults.
|
||||||
|
- local `audio_dir` resolves relative to `session.yml`.
|
||||||
|
|
||||||
|
Minimal local-file usage:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03
|
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03
|
||||||
```
|
```
|
||||||
|
|
||||||
Previous-session-enabled variant:
|
Previous-session-enabled variant:
|
||||||
@@ -92,19 +135,17 @@ Previous-session-enabled variant:
|
|||||||
```yaml
|
```yaml
|
||||||
session_id: "{{ session_id }}"
|
session_id: "{{ session_id }}"
|
||||||
previous_session_id: "{{ previous_session_id }}"
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
```
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||||
```
|
```
|
||||||
|
|
||||||
## 6. Production-oriented config
|
## 5. Production-oriented config set
|
||||||
|
|
||||||
|
### `pipeline.yml`
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
workspace:
|
workspace:
|
||||||
@@ -124,6 +165,10 @@ spool:
|
|||||||
root: /var/spool/narratio
|
root: /var/spool/narratio
|
||||||
delete_audio_after_archive: true
|
delete_audio_after_archive: true
|
||||||
|
|
||||||
|
cache:
|
||||||
|
root: /var/cache/narratio
|
||||||
|
s3_audio: true
|
||||||
|
|
||||||
archive:
|
archive:
|
||||||
enabled: true
|
enabled: true
|
||||||
upload_run: true
|
upload_run: true
|
||||||
@@ -134,6 +179,9 @@ archive:
|
|||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
dest: artifacts/session_recap.md
|
dest: artifacts/session_recap.md
|
||||||
required: true
|
required: true
|
||||||
|
locks:
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
reason: Final recap was manually edited.
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
@@ -153,14 +201,53 @@ scriptorium:
|
|||||||
required: false
|
required: false
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `campaign.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
speakers_file: /srv/narratio/campaigns/forsaken/speakers.yml
|
||||||
|
autocorrect_file: /srv/narratio/campaigns/forsaken/autocorrect.yml
|
||||||
|
glossary_file: /srv/narratio/campaigns/forsaken/glossary.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Local `session.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
session_id: "{{ session_id }}"
|
||||||
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
|
date: 2026-05-03
|
||||||
|
title: The Black Cabin
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
```
|
||||||
|
|
||||||
|
### S3-first session config
|
||||||
|
|
||||||
|
For S3-first operation, upload the same `session.yml` content to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Then run with explicit or discovered pipeline/campaign config and no `--session`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --config /usr/local/etc/narratio/pipeline.yml --campaign /usr/local/etc/narratio/campaign.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||||
|
```
|
||||||
|
|
||||||
Operational notes:
|
Operational notes:
|
||||||
|
|
||||||
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
||||||
- `source` is required; `dest` is optional and derived when omitted.
|
- `source` is required; `dest` is optional and derived when omitted.
|
||||||
|
- `archive.locks` skips top-level promotion overwrites for static locked sources while preserving run-local uploads.
|
||||||
|
- operator-created mutable locks are stored at `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml` and are merged with static locks.
|
||||||
- Narratio does not auto-promote all generated analyze artifacts.
|
- Narratio does not auto-promote all generated analyze artifacts.
|
||||||
- `restore` reads the same config/session inputs and restore scope is bounded by committed archive current state.
|
- `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
|
||||||
|
- `clean` removes workspace/spool state by default and preserves `pipeline.cache.root` unless `--clear-cache` is passed.
|
||||||
|
|
||||||
## 7. Full pipeline reference
|
## 6. Full pipeline reference
|
||||||
|
|
||||||
| Path | Type | Required | Default |
|
| Path | Type | Required | Default |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
@@ -179,12 +266,17 @@ Operational notes:
|
|||||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||||
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
|
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
|
||||||
|
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||||
|
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||||
| `pipeline.archive.enabled` | bool | No | `true` |
|
| `pipeline.archive.enabled` | bool | No | `true` |
|
||||||
| `pipeline.archive.upload_run` | bool | No | `true` |
|
| `pipeline.archive.upload_run` | bool | No | `true` |
|
||||||
| `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule |
|
| `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule |
|
||||||
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
||||||
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
||||||
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
||||||
|
| `pipeline.archive.locks[]` | list | No | empty |
|
||||||
|
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
|
||||||
|
| `pipeline.archive.locks[].reason` | string | No | empty |
|
||||||
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
||||||
| `pipeline.whisperx.language` | string | No | `en` |
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
||||||
@@ -285,6 +377,8 @@ Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
|||||||
- `narratio.bounds.session`
|
- `narratio.bounds.session`
|
||||||
- `narratio.artifact.<configured_artifact_key>`
|
- `narratio.artifact.<configured_artifact_key>`
|
||||||
|
|
||||||
|
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
|
||||||
|
|
||||||
Archive promotion destination rules:
|
Archive promotion destination rules:
|
||||||
|
|
||||||
- `dest` must be a clean relative path (not absolute, no traversal).
|
- `dest` must be a clean relative path (not absolute, no traversal).
|
||||||
@@ -294,10 +388,39 @@ Archive promotion destination rules:
|
|||||||
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
||||||
- derivation failure is a config validation error.
|
- derivation failure is a config validation error.
|
||||||
|
|
||||||
|
Archive lock rules:
|
||||||
|
|
||||||
|
- locks are source-based and do not accept `dest`.
|
||||||
|
- duplicate lock sources are rejected.
|
||||||
|
- static `pipeline.archive.locks` win over remote mutable locks for the same source.
|
||||||
|
- locked promotions are recorded as intentional skips in archive metadata.
|
||||||
|
- locked required promotions do not fail archive by default.
|
||||||
|
- ordinary `--force` reruns do not override locks.
|
||||||
|
|
||||||
|
Remote mutable lock store:
|
||||||
|
|
||||||
|
- path: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml`.
|
||||||
|
- strict YAML shape: top-level `locks`, each with `source` and optional `reason`.
|
||||||
|
- `narratio locks add` and `narratio locks remove` mutate only the remote lock store.
|
||||||
|
- writes use existence checks plus `--force` for updates; they are not compare-and-swap atomic.
|
||||||
|
|
||||||
Restore-related implications:
|
Restore-related implications:
|
||||||
|
|
||||||
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
||||||
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
|
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
|
||||||
|
- S3 audio downloads use `pipeline.spool.root` for active downloads and `pipeline.cache.root` for reusable cached audio when `pipeline.cache.s3_audio` is true.
|
||||||
|
- `pipeline.cache.root` is durable local cache state. It is not workspace state and is preserved by default by `narratio clean`.
|
||||||
|
|
||||||
|
## 7. Full campaign reference
|
||||||
|
|
||||||
|
| Path | Type | Required | Default |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `campaign.campaign` | string | Yes | none |
|
||||||
|
| `campaign.inputs.speakers_file` | string | Yes | none |
|
||||||
|
| `campaign.inputs.autocorrect_file` | string | Yes | none |
|
||||||
|
| `campaign.inputs.glossary_file` | string | Yes | none |
|
||||||
|
|
||||||
|
Campaign input paths may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
|
||||||
|
|
||||||
## 8. Full session reference
|
## 8. Full session reference
|
||||||
|
|
||||||
@@ -305,15 +428,17 @@ Restore-related implications:
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `session.session_id` | string | Yes | none |
|
| `session.session_id` | string | Yes | none |
|
||||||
| `session.previous_session_id` | string | No | empty |
|
| `session.previous_session_id` | string | No | empty |
|
||||||
| `session.campaign` | string | Yes | none |
|
| `session.campaign` | string | No | `campaign.campaign` |
|
||||||
| `session.date` | string | No | empty |
|
| `session.date` | string | No | empty |
|
||||||
| `session.title` | string | No | empty |
|
| `session.title` | string | No | empty |
|
||||||
| `session.inputs.audio_dir` | string | Conditional | empty |
|
| `session.inputs.audio_dir` | string | Conditional | empty |
|
||||||
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
||||||
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
||||||
| `session.inputs.speakers_file` | string | Yes | none |
|
| `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
|
||||||
| `session.inputs.autocorrect_file` | string | Yes | none |
|
| `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
|
||||||
| `session.inputs.glossary_file` | string | Yes | none |
|
| `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
|
||||||
|
|
||||||
|
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
|
||||||
|
|
||||||
Audio-source rule:
|
Audio-source rule:
|
||||||
|
|
||||||
@@ -354,6 +479,7 @@ Maintained examples:
|
|||||||
- `examples/pipeline.minimal.yml`
|
- `examples/pipeline.minimal.yml`
|
||||||
- `examples/pipeline.production.yml`
|
- `examples/pipeline.production.yml`
|
||||||
- `examples/pipeline.full.annotated.yml`
|
- `examples/pipeline.full.annotated.yml`
|
||||||
|
- `examples/campaign.yml`
|
||||||
- `examples/session.template.yml`
|
- `examples/session.template.yml`
|
||||||
- `examples/session.local-audio.yml`
|
- `examples/session.local-audio.yml`
|
||||||
- `examples/session.s3-audio.yml`
|
- `examples/session.s3-audio.yml`
|
||||||
|
|||||||
@@ -69,6 +69,8 @@ For design principles and invariants, see [docs/architecture.md](./architecture.
|
|||||||
2. Add or update command tests (`TestExecute` and command-specific tests).
|
2. Add or update command tests (`TestExecute` and command-specific tests).
|
||||||
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
||||||
|
|
||||||
|
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
|
||||||
|
|
||||||
### Add or modify stages/adapters
|
### Add or modify stages/adapters
|
||||||
|
|
||||||
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ Current execution usage:
|
|||||||
|
|
||||||
Default construction in app runner:
|
Default construction in app runner:
|
||||||
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
|
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
|
||||||
|
- Object-store construction goes through app command orchestration so configured filesystem secrets are loaded before the storage adapter is initialized.
|
||||||
- Callers can inject test/fake implementations through `app.RunOptions.Env`.
|
- Callers can inject test/fake implementations through `app.RunOptions.Env`.
|
||||||
|
|
||||||
## State and manifest behavior
|
## State and manifest behavior
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ Outputs:
|
|||||||
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
||||||
- runtime catalog entries for built-ins and configured artifacts;
|
- runtime catalog entries for built-ins and configured artifacts;
|
||||||
- requirement sets for canonical previous-session inputs.
|
- requirement sets for canonical previous-session inputs.
|
||||||
|
- canonical S3 session, run, current, session config, session locks, audio, and promoted artifact keys.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
@@ -43,6 +44,15 @@ Does not own:
|
|||||||
- configured artifact: `narratio.artifact.<artifact_key>`
|
- configured artifact: `narratio.artifact.<artifact_key>`
|
||||||
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
|
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
|
## S3 key helpers
|
||||||
|
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||||
|
- session config: `{session_prefix}/session.yml`
|
||||||
|
- session lock store: `{session_prefix}/locks.yml`
|
||||||
|
- run prefix: `{session_prefix}/runs/{run_id}/`
|
||||||
|
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
|
||||||
|
- current manifest: `{session_prefix}/current/manifest.json`
|
||||||
|
- current run pointer: `{session_prefix}/current/run_id.txt`
|
||||||
|
|
||||||
## Runtime catalog model
|
## Runtime catalog model
|
||||||
Catalog entries track:
|
Catalog entries track:
|
||||||
- `planned`: source is registered for this run;
|
- `planned`: source is registered for this run;
|
||||||
@@ -61,6 +71,7 @@ Previous-session canonical provenance values include:
|
|||||||
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
||||||
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
||||||
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
||||||
|
- Archive-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
|
||||||
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
||||||
|
|
||||||
## Previous-session requirement scanning
|
## Previous-session requirement scanning
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ Does not own:
|
|||||||
- `pipeline.workspace.root` (local restore target root).
|
- `pipeline.workspace.root` (local restore target root).
|
||||||
- `pipeline.storage.*` (remote backend + archive identity derivation).
|
- `pipeline.storage.*` (remote backend + archive identity derivation).
|
||||||
- `pipeline.storage.s3.*` identity components used by archive prefix helpers.
|
- `pipeline.storage.s3.*` identity components used by archive prefix helpers.
|
||||||
|
- `pipeline.spool.root` for active audio downloads.
|
||||||
|
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
|
|
||||||
@@ -48,6 +50,7 @@ Does not own:
|
|||||||
- `current/run_id.txt` must exist and be non-empty.
|
- `current/run_id.txt` must exist and be non-empty.
|
||||||
- `current/manifest.json` must decode and match requested session/campaign.
|
- `current/manifest.json` must decode and match requested session/campaign.
|
||||||
- Non-dry-run writes restore files to canonical session paths.
|
- Non-dry-run writes restore files to canonical session paths.
|
||||||
|
- With `--include-audio`, restore uses the shared S3 audio cache for `audio/**` objects. Cache hits avoid object downloads; cache misses download through spool, install the work file, and populate cache.
|
||||||
- Manifest install behavior:
|
- Manifest install behavior:
|
||||||
- validated before replacement.
|
- validated before replacement.
|
||||||
- installed last among download actions.
|
- installed last among download actions.
|
||||||
@@ -72,6 +75,7 @@ Restore path scope:
|
|||||||
## Skip and resume behavior
|
## Skip and resume behavior
|
||||||
- Restore does not participate in stage skip/resume decisions.
|
- Restore does not participate in stage skip/resume decisions.
|
||||||
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
|
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
|
||||||
|
- Audio cache is outside the workspace and is reused across restore and prepare invocations.
|
||||||
- Dry-run is read-only and returns plan output only.
|
- Dry-run is read-only and returns plan output only.
|
||||||
|
|
||||||
## Failure behavior
|
## Failure behavior
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Inputs:
|
|||||||
- session manifest and prerequisite stage records
|
- session manifest and prerequisite stage records
|
||||||
- run root contents under `runs/{run_id}/`
|
- run root contents under `runs/{run_id}/`
|
||||||
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
||||||
|
- effective source-based promotion locks from static config and remote session lock store
|
||||||
- session-level `previous/**` cache files when present
|
- session-level `previous/**` cache files when present
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
@@ -23,6 +24,7 @@ Owns:
|
|||||||
- Prerequisite stage success enforcement
|
- Prerequisite stage success enforcement
|
||||||
- Run file collection and upload (excluding `audio/`)
|
- Run file collection and upload (excluding `audio/`)
|
||||||
- Promotion rule resolution and upload
|
- Promotion rule resolution and upload
|
||||||
|
- Promotion lock enforcement
|
||||||
- Session previous-cache file collection/upload
|
- Session previous-cache file collection/upload
|
||||||
- Commit pointer publish order
|
- Commit pointer publish order
|
||||||
|
|
||||||
@@ -34,6 +36,8 @@ Does not own:
|
|||||||
- `pipeline.archive.enabled`
|
- `pipeline.archive.enabled`
|
||||||
- `pipeline.archive.upload_run`
|
- `pipeline.archive.upload_run`
|
||||||
- `pipeline.archive.promote_artifacts`
|
- `pipeline.archive.promote_artifacts`
|
||||||
|
- `pipeline.archive.locks`
|
||||||
|
- `{session_prefix}/locks.yml` loaded by app orchestration before archive execution
|
||||||
- `pipeline.storage.s3.bucket`
|
- `pipeline.storage.s3.bucket`
|
||||||
- `pipeline.storage.s3.root_prefix`
|
- `pipeline.storage.s3.root_prefix`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
@@ -47,9 +51,12 @@ Does not own:
|
|||||||
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
||||||
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
||||||
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
||||||
|
- Skips top-level promotion uploads for effective locked sources; run-local uploads still publish.
|
||||||
|
- Effective locks are the union of `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
|
||||||
- Writes metadata including:
|
- Writes metadata including:
|
||||||
- upload counts/paths
|
- upload counts/paths
|
||||||
- `previous_files_uploaded` and `previous_uploaded_paths`
|
- `previous_files_uploaded` and `previous_uploaded_paths`
|
||||||
|
- `locked_promotion_count` and `locked_promotions`
|
||||||
- `current_manifest_key`
|
- `current_manifest_key`
|
||||||
- `current_run_id_key`
|
- `current_run_id_key`
|
||||||
- `current_pointer_written`
|
- `current_pointer_written`
|
||||||
@@ -60,7 +67,8 @@ Does not own:
|
|||||||
- Runner-level skip also applies for previously succeeded stage unless forced.
|
- Runner-level skip also applies for previously succeeded stage unless forced.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
- Fails on missing prerequisite success, missing object store when required, missing run root, missing required promotion source, upload failures, or pointer write failures.
|
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
|
||||||
|
- Locked required promotions are intentional skips and do not fail archive.
|
||||||
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Tests to Inspect Before Changing
|
||||||
@@ -70,5 +78,7 @@ Does not own:
|
|||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Run upload excludes `audio/` subtree.
|
- Run upload excludes `audio/` subtree.
|
||||||
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
|
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
|
||||||
|
- Ordinary `--force` does not override archive locks.
|
||||||
|
- Malformed or unreadable remote lock store fails archive-capable execution before promotion.
|
||||||
- `current/manifest.json` uploads before `current/run_id.txt`.
|
- `current/manifest.json` uploads before `current/run_id.txt`.
|
||||||
- `current/run_id.txt` is the remote publish commit marker.
|
- `current/run_id.txt` is the remote publish commit marker.
|
||||||
|
|||||||
@@ -10,8 +10,9 @@ Prepare owns:
|
|||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- resolved config/session (`pipeline.yml`, `session.yml`);
|
- resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
|
||||||
- session-local input files (`speakers`, `autocorrect`, `glossary`);
|
- remote session provenance when `session.yml` was loaded from S3;
|
||||||
|
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
|
||||||
- audio source:
|
- audio source:
|
||||||
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
||||||
- S3: `session.inputs.audio_s3.prefix`;
|
- S3: `session.inputs.audio_s3.prefix`;
|
||||||
@@ -19,6 +20,7 @@ Inputs:
|
|||||||
- remote previous-session current archive state when previous hydration is required.
|
- remote previous-session current archive state when previous hydration is required.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
|
- `inputs/campaign.yml`;
|
||||||
- `inputs/session.yml`;
|
- `inputs/session.yml`;
|
||||||
- `inputs/pipeline.resolved.yml`;
|
- `inputs/pipeline.resolved.yml`;
|
||||||
- `inputs/speakers.yml`;
|
- `inputs/speakers.yml`;
|
||||||
@@ -53,11 +55,17 @@ Does not own:
|
|||||||
- `session.inputs.audio_s3.prefix`
|
- `session.inputs.audio_s3.prefix`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
- `pipeline.spool.root`
|
- `pipeline.spool.root`
|
||||||
|
- `pipeline.cache.root`
|
||||||
|
- `pipeline.cache.s3_audio`
|
||||||
- `pipeline.storage.s3.bucket`
|
- `pipeline.storage.s3.bucket`
|
||||||
- `pipeline.storage.s3.root_prefix`
|
- `pipeline.storage.s3.root_prefix`
|
||||||
- `pipeline.scriptorium.artifacts.<name>.enabled`
|
- `pipeline.scriptorium.artifacts.<name>.enabled`
|
||||||
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
|
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
|
||||||
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
|
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
|
||||||
|
- `campaign.campaign`
|
||||||
|
- `campaign.inputs.speakers_file`
|
||||||
|
- `campaign.inputs.autocorrect_file`
|
||||||
|
- `campaign.inputs.glossary_file`
|
||||||
|
|
||||||
## External adapters used
|
## External adapters used
|
||||||
- `storage.ObjectStore` for:
|
- `storage.ObjectStore` for:
|
||||||
@@ -67,15 +75,21 @@ Does not own:
|
|||||||
## State and manifest behavior
|
## State and manifest behavior
|
||||||
- Ensures workspace layout exists.
|
- Ensures workspace layout exists.
|
||||||
- Materializes canonical input files and audio files.
|
- Materializes canonical input files and audio files.
|
||||||
|
- For S3 audio, uses run-scoped spool for active downloads and durable cache for reusable audio files; cache hits copy directly to work audio without downloading the object again.
|
||||||
|
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
|
||||||
|
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
|
||||||
|
- Resolves session-provided stable input overrides relative to `session.yml`.
|
||||||
- Scans enabled configured artifact inputs for canonical sources:
|
- Scans enabled configured artifact inputs for canonical sources:
|
||||||
- `narratio.previous_session.artifact.<artifact_key>`
|
- `narratio.previous_session.artifact.<artifact_key>`
|
||||||
- If one or more canonical previous-session requirements exist:
|
- If one or more canonical previous-session requirements exist:
|
||||||
- clears managed `previous/` state;
|
- clears managed `previous/` state;
|
||||||
- hydrates required/optional previous artifacts from the configured previous session’s committed archive current state;
|
- hydrates required/optional previous artifacts from the configured previous session’s committed archive current state;
|
||||||
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
||||||
|
- stores archive-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
|
||||||
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
|
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
|
||||||
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
||||||
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
||||||
|
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
|
||||||
|
|
||||||
## Required and optional previous-session behavior
|
## Required and optional previous-session behavior
|
||||||
- `previous_session_id` unset:
|
- `previous_session_id` unset:
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ Document Narratio's remote storage backend contracts and implementations under `
|
|||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- Resolved storage config (`pipeline.storage.*`).
|
- Resolved storage config (`pipeline.storage.*`).
|
||||||
- Bucket-relative object keys and local file paths from stage/app orchestration.
|
- Already-loaded environment variables for configured S3 credentials.
|
||||||
|
- Bucket-relative object keys and local file paths from app/stage orchestration.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
|
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
|
||||||
@@ -22,6 +23,7 @@ Does not own:
|
|||||||
- Session/run prefix semantics.
|
- Session/run prefix semantics.
|
||||||
- Archive commit order semantics.
|
- Archive commit order semantics.
|
||||||
- Manifest updates.
|
- Manifest updates.
|
||||||
|
- Filesystem secret loading from `pipeline.secrets.env_dir`.
|
||||||
|
|
||||||
## Config fields used
|
## Config fields used
|
||||||
- `pipeline.storage.backend`
|
- `pipeline.storage.backend`
|
||||||
@@ -55,8 +57,10 @@ Implementations:
|
|||||||
## Failure behavior
|
## Failure behavior
|
||||||
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
|
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
|
||||||
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
|
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
|
||||||
|
- App command orchestration loads configured filesystem secrets before calling the object-store factory.
|
||||||
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
|
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
|
||||||
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
|
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
|
||||||
|
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
|
||||||
|
|
||||||
## Tests to inspect before changing
|
## Tests to inspect before changing
|
||||||
- `internal/adapters/storage/factory_test.go`
|
- `internal/adapters/storage/factory_test.go`
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ Does not own:
|
|||||||
- `pipeline.workspace.cleanup_after_archive`
|
- `pipeline.workspace.cleanup_after_archive`
|
||||||
- `pipeline.spool.root`
|
- `pipeline.spool.root`
|
||||||
- `pipeline.spool.delete_audio_after_archive`
|
- `pipeline.spool.delete_audio_after_archive`
|
||||||
|
- `pipeline.cache.root`
|
||||||
|
- `pipeline.cache.s3_audio`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
|
|
||||||
@@ -43,7 +45,11 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
|
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
|
||||||
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
|
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
|
||||||
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
|
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
|
||||||
- For S3 audio sessions, `prepare` records spool/work paths and S3 provenance in `manifest.Inputs`.
|
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
|
||||||
|
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
|
||||||
|
- `narratio clean --session-id <id>` removes the session work root and session spool root.
|
||||||
|
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
|
||||||
|
- `narratio clean --clear-cache` is the explicit opt-in for deleting matching S3 audio cache entries.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and Resume Behavior
|
||||||
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
|
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
|
||||||
@@ -67,4 +73,5 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
- Run roots are always nested: `runs/{run_id}` under the session root.
|
- Run roots are always nested: `runs/{run_id}` under the session root.
|
||||||
- Run-local output promotion must end in canonical session paths.
|
- Run-local output promotion must end in canonical session paths.
|
||||||
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
||||||
- Cleanup only targets run-scoped directories and must never delete configured root directories.
|
- Automatic post-archive cleanup only targets run-scoped directories and must never delete configured root directories.
|
||||||
|
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.
|
||||||
|
|||||||
@@ -6,21 +6,37 @@ For field-level configuration, see [docs/config.md](./config.md). For full comma
|
|||||||
|
|
||||||
## Normal workflow (S3-first path)
|
## Normal workflow (S3-first path)
|
||||||
|
|
||||||
1. Upload session `.flac` files to object storage under the configured session audio prefix.
|
1. Create or upload `session.yml`, or pass a local `session.yml` explicitly.
|
||||||
2. Run Narratio:
|
2. Upload session `.flac` files to object storage under the configured session audio prefix.
|
||||||
|
3. Run Narratio:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --session-id 2026-04-04
|
narratio run --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
3. Read success output:
|
4. Read success output:
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||||
- use `manifest=<path>` with `status` for inspection.
|
- use `manifest=<path>` with `status` for inspection.
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
- default config/session discovery applies unless `--config` and `--session` are passed.
|
- default config/campaign/session discovery checks system config locations unless `--config`, `--campaign`, and `--session` are passed.
|
||||||
|
- when local `session.yml` discovery misses, `--session-id` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
||||||
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
||||||
|
|
||||||
|
Initialize a remote session skeleton:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init --config /etc/narratio/pipeline.yml --campaign /etc/narratio/campaign.yml --session-id 2026-04-04 --remote
|
||||||
|
```
|
||||||
|
|
||||||
|
Remote init writes `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`. It fails if the object already exists unless `--force` is passed.
|
||||||
|
|
||||||
|
Validate before running:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate --session-id 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
## Restore workflow
|
## Restore workflow
|
||||||
|
|
||||||
Use restore when local durable session state is missing or stale and archive current state is authoritative.
|
Use restore when local durable session state is missing or stale and archive current state is authoritative.
|
||||||
@@ -40,7 +56,7 @@ narratio restore --session-id 2026-04-04
|
|||||||
Post-restore analyze rerun pattern:
|
Post-restore analyze rerun pattern:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage --session-id 2026-04-04 --force analyze
|
narratio analyze --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Restore source-of-truth:
|
Restore source-of-truth:
|
||||||
@@ -52,6 +68,16 @@ Restore default scope:
|
|||||||
- includes `audio/**` only with `--include-audio`
|
- includes `audio/**` only with `--include-audio`
|
||||||
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
||||||
|
|
||||||
|
Reset local state before restore testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean --session-id 2026-04-04 --dry-run
|
||||||
|
narratio clean --session-id 2026-04-04
|
||||||
|
narratio restore --session-id 2026-04-04 --include-audio
|
||||||
|
```
|
||||||
|
|
||||||
|
`clean` removes the local session work directory and session spool directory. It preserves the durable S3 audio cache by default, so repeated restore or forced prepare tests do not re-download large audio files.
|
||||||
|
|
||||||
## Local filesystem layout and state artifacts
|
## Local filesystem layout and state artifacts
|
||||||
|
|
||||||
Session root:
|
Session root:
|
||||||
@@ -61,6 +87,7 @@ Primary state:
|
|||||||
- `manifest.json`: session-level stage state.
|
- `manifest.json`: session-level stage state.
|
||||||
- `runs/{run_id}/manifest.json`: invocation-level state.
|
- `runs/{run_id}/manifest.json`: invocation-level state.
|
||||||
- `.lock`: session lock while a modifying command is active.
|
- `.lock`: session lock while a modifying command is active.
|
||||||
|
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
|
||||||
|
|
||||||
Canonical session directories:
|
Canonical session directories:
|
||||||
- `inputs/`
|
- `inputs/`
|
||||||
@@ -97,8 +124,9 @@ Configured artifact source reuse:
|
|||||||
- reused configured artifact provenance is `filesystem.disabled_artifact_output`.
|
- reused configured artifact provenance is `filesystem.disabled_artifact_output`.
|
||||||
|
|
||||||
`--artifacts` behavior:
|
`--artifacts` behavior:
|
||||||
- accepted on `run`, `resume`, and `run-stage analyze`.
|
- accepted on `run`, `resume`, `run-stage analyze`, and `analyze`.
|
||||||
- filters analyze execution only; does not force stage rerun.
|
- filters analyze execution only.
|
||||||
|
- does not imply force on `run`, `resume`, or `run-stage`; `narratio analyze` is force-by-design.
|
||||||
|
|
||||||
Canonical previous-session input behavior:
|
Canonical previous-session input behavior:
|
||||||
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
|
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
|
||||||
@@ -116,6 +144,7 @@ When archive is enabled and run upload is enabled, archive publishes under:
|
|||||||
Archive uploads:
|
Archive uploads:
|
||||||
- run record files from run root (excluding `audio/`).
|
- run record files from run root (excluding `audio/`).
|
||||||
- promoted files from explicit `archive.promote_artifacts` rules.
|
- promoted files from explicit `archive.promote_artifacts` rules.
|
||||||
|
- mutable session locks from helper commands live at `{session_prefix}/locks.yml`.
|
||||||
|
|
||||||
Publish order:
|
Publish order:
|
||||||
1. upload `current/manifest.json`
|
1. upload `current/manifest.json`
|
||||||
@@ -129,6 +158,17 @@ Archive promotion is explicit and source-based:
|
|||||||
- missing required promotion sources fail archive stage.
|
- missing required promotion sources fail archive stage.
|
||||||
- missing optional promotion sources are skipped.
|
- missing optional promotion sources are skipped.
|
||||||
- invalid resolved artifacts fail archive stage.
|
- invalid resolved artifacts fail archive stage.
|
||||||
|
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
|
||||||
|
- remote locks from `{session_prefix}/locks.yml` are merged with static `archive.locks`; static locks win on duplicate sources.
|
||||||
|
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
|
||||||
|
|
||||||
|
Lock helper behavior:
|
||||||
|
- `narratio locks --session-id <id>` lists effective static and remote locks.
|
||||||
|
- `narratio locks add --session-id <id> --reason <text> <source>` writes a remote lock.
|
||||||
|
- `narratio locks add --session-id <id> --force --reason <text> <source>` updates an existing remote lock reason.
|
||||||
|
- `narratio locks remove --session-id <id> <source>` removes only a remote lock.
|
||||||
|
- `locks remove` cannot remove static pipeline locks.
|
||||||
|
- remote lock writes check whether the lock store exists, but are not compare-and-swap atomic.
|
||||||
|
|
||||||
## Resume, retry, restore, and safe rerun behavior
|
## Resume, retry, restore, and safe rerun behavior
|
||||||
|
|
||||||
@@ -146,6 +186,7 @@ Restore conflict policy:
|
|||||||
|
|
||||||
Forced reruns:
|
Forced reruns:
|
||||||
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
||||||
|
- ordinary `--force` does not override archive locks.
|
||||||
|
|
||||||
Safe rerun pattern:
|
Safe rerun pattern:
|
||||||
1. rerun the changed stage with `--force`.
|
1. rerun the changed stage with `--force`.
|
||||||
@@ -153,12 +194,20 @@ Safe rerun pattern:
|
|||||||
|
|
||||||
## Cleanup behavior
|
## Cleanup behavior
|
||||||
|
|
||||||
Cleanup is considered only when archive stage executed and succeeded.
|
Automatic post-archive cleanup is considered only when archive stage executed and succeeded.
|
||||||
|
|
||||||
Cleanup toggles:
|
Automatic cleanup toggles:
|
||||||
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
|
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
|
||||||
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
|
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
|
||||||
|
|
||||||
|
Manual cleanup:
|
||||||
|
- `narratio clean --session-id <id>` deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
|
||||||
|
- `narratio clean --all` deletes all local session work under `{workspace.root}/work` and all spool children under `{spool.root}`.
|
||||||
|
- `--dry-run` prints targets without deleting.
|
||||||
|
- `--clear-cache` also removes matching S3 audio cache files. Without it, cache is preserved.
|
||||||
|
|
||||||
|
The S3 audio cache under `pipeline.cache.root` is durable input cache state, not workspace or spool state. Automatic cleanup and default manual cleanup do not delete it.
|
||||||
|
|
||||||
Cleanup eligibility gates:
|
Cleanup eligibility gates:
|
||||||
- archive enabled
|
- archive enabled
|
||||||
- archive run upload enabled
|
- archive run upload enabled
|
||||||
@@ -185,18 +234,26 @@ Recommended recovery:
|
|||||||
|
|
||||||
1. inspect state:
|
1. inspect state:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio status --session-id 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
This reports local manifest state, committed remote current state, expected remote transcript/artifact availability, and archive locks.
|
||||||
|
|
||||||
|
2. for one manifest file, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio status --manifest <manifest-path>
|
narratio status --manifest <manifest-path>
|
||||||
```
|
```
|
||||||
|
|
||||||
2. for restore-specific checks, run:
|
3. for restore-specific checks, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore --session-id 2026-04-04 --dry-run
|
narratio restore --session-id 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
3. fix root cause (config/input/credentials/storage/service availability).
|
4. fix root cause (config/input/credentials/storage/service availability).
|
||||||
4. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
|
5. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
|
||||||
|
|
||||||
## Restore report
|
## Restore report
|
||||||
|
|
||||||
@@ -213,7 +270,9 @@ Dry-run does not write restore report files.
|
|||||||
|
|
||||||
## Operational caveats
|
## Operational caveats
|
||||||
|
|
||||||
- `status` requires explicit `--manifest`; there is no session-id lookup command.
|
- `status` with no config/session flags still requires explicit `--manifest`.
|
||||||
|
- `status --session-id <id>` uses normal config/session loading, including remote session fallback.
|
||||||
|
- `status --session-id <id>` includes the same promoted remote output availability view as `artifacts list --remote` when storage is configured.
|
||||||
- local and S3 audio input modes are mutually exclusive.
|
- local and S3 audio input modes are mutually exclusive.
|
||||||
- archive publish requires upstream stages through `analyze` to be `succeeded`.
|
- archive publish requires upstream stages through `analyze` to be `succeeded`.
|
||||||
- required promotion rules can fail when selected analyze artifacts did not generate a required file path.
|
- required promotion rules can fail when selected analyze artifacts did not generate a required file path.
|
||||||
|
|||||||
52
docs/roadmap/operations.md
Normal file
52
docs/roadmap/operations.md
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
# Roadmap: Operator Helper Commands
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Implemented.
|
||||||
|
|
||||||
|
The operator helper command set is no longer conceptual. Current behavior is documented in:
|
||||||
|
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/config.md`
|
||||||
|
- `docs/internal/artifacts.md`
|
||||||
|
- `docs/internal/stage-archive.md`
|
||||||
|
|
||||||
|
## Implemented Commands
|
||||||
|
|
||||||
|
- `narratio session validate`
|
||||||
|
- `narratio status --manifest <path>`
|
||||||
|
- `narratio status --session-id <id>`
|
||||||
|
- `narratio session init --output <path>`
|
||||||
|
- `narratio session init --remote`
|
||||||
|
- `narratio artifacts list`
|
||||||
|
- `narratio artifacts list --remote`
|
||||||
|
- `narratio locks`
|
||||||
|
- `narratio locks add <source>`
|
||||||
|
- `narratio locks remove <source>`
|
||||||
|
|
||||||
|
## Implemented Decisions
|
||||||
|
|
||||||
|
- Helper output is text-only. No JSON schema exists yet.
|
||||||
|
- `status` remains a top-level command.
|
||||||
|
- `session validate`, `session init`, and `artifacts list` are nested helper commands.
|
||||||
|
- `locks` is the single top-level command for listing, adding, and removing archive promotion locks.
|
||||||
|
- Remote session initialization requires explicit `--remote`.
|
||||||
|
- Local session initialization requires `--output`.
|
||||||
|
- Remote artifact availability is opt-in with `artifacts list --remote`.
|
||||||
|
- Mutable locks are source-based and stored at `{session_prefix}/locks.yml`.
|
||||||
|
- The remote lock store uses strict YAML with top-level `locks`.
|
||||||
|
- Static `pipeline.archive.locks` and remote locks are merged; static locks win on duplicate sources.
|
||||||
|
- `locks remove` removes only remote locks.
|
||||||
|
- Ordinary execution `--force` does not override locks.
|
||||||
|
- Remote lock writes use existence checks and `--force` for updates; there is no compare-and-swap protection.
|
||||||
|
|
||||||
|
## Remaining Future Enhancements
|
||||||
|
|
||||||
|
These are intentionally not implemented:
|
||||||
|
|
||||||
|
- `--json` output for helper commands.
|
||||||
|
- Optimistic concurrency or ETag compare-and-swap for remote lock mutations.
|
||||||
|
- Rich remote artifact availability across historical run-local objects.
|
||||||
|
- Session-lock acquisition for remote mutation helpers.
|
||||||
|
- Broader campaign helper commands such as `campaign validate` or `campaign publish`.
|
||||||
@@ -1,629 +0,0 @@
|
|||||||
# Roadmap: Previous-Session Artifacts
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Completed.
|
|
||||||
|
|
||||||
This roadmap describes the implementation strategy for first-class previous-session artifact support in Narratio. It belongs under `docs/roadmap/previous.md` until the feature is implemented. After implementation, current behavior should be documented in the appropriate user-facing and internal documentation files, and this roadmap should be removed or marked complete according to the documentation policy.
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Narratio should support using artifacts from a previous session as inputs to artifacts generated for the current session.
|
|
||||||
|
|
||||||
The primary use case is session recap continuity: a current session recap should be able to consume the previous session recap. The design should support arbitrary previous-session artifacts from the start, not just `session_recap`.
|
|
||||||
|
|
||||||
The canonical source syntax should be:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
source: narratio.previous_session.artifact.<artifact_name>
|
|
||||||
```
|
|
||||||
|
|
||||||
For example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
source: narratio.previous_session.artifact.session_recap
|
|
||||||
```
|
|
||||||
|
|
||||||
Previous-session artifacts are materialized during the `prepare` stage into a current-session top-level `previous/` directory. Downstream stages consume only the local `previous/` copies. The S3 backend is authoritative for previous-session state.
|
|
||||||
|
|
||||||
## Design Decisions
|
|
||||||
|
|
||||||
### 1. Add `previous_session_id` to `session.yml`
|
|
||||||
|
|
||||||
Add an optional top-level session key:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
session_id: "{{ session_id }}"
|
|
||||||
previous_session_id: "{{ previous_session_id }}"
|
|
||||||
campaign: sample-campaign
|
|
||||||
```
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- `previous_session_id` is optional.
|
|
||||||
- If present, it identifies the previous session within the same campaign.
|
|
||||||
- It must not equal `session_id`.
|
|
||||||
- It should use the same validation rules as `session_id`.
|
|
||||||
- It may be supplied through session templating.
|
|
||||||
- Add a CLI/template value such as `--previous-session-id <id>` if required by the existing session templating implementation.
|
|
||||||
- If a template placeholder for `previous_session_id` is present and no value is supplied, loading should fail with a clear unresolved-template error.
|
|
||||||
|
|
||||||
### 2. Use canonical previous-session artifact source IDs
|
|
||||||
|
|
||||||
Support this source pattern in Scriptorium artifact input definitions:
|
|
||||||
|
|
||||||
```text
|
|
||||||
narratio.previous_session.artifact.<artifact_name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
previous_recap:
|
|
||||||
source: narratio.previous_session.artifact.session_recap
|
|
||||||
required: false
|
|
||||||
|
|
||||||
previous_quest_log:
|
|
||||||
source: narratio.previous_session.artifact.quest_log
|
|
||||||
required: true
|
|
||||||
```
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- `<artifact_name>` must be a valid configured artifact key.
|
|
||||||
- Use the same artifact key validation rules as current-session runtime artifacts.
|
|
||||||
- Do not special-case `session_recap`.
|
|
||||||
- Do not limit implementation to a fixed list of previous artifacts.
|
|
||||||
|
|
||||||
### 3. Add a top-level `previous/` workspace directory
|
|
||||||
|
|
||||||
Extend the session workspace layout with:
|
|
||||||
|
|
||||||
```text
|
|
||||||
previous/
|
|
||||||
manifest.json
|
|
||||||
artifacts/
|
|
||||||
<artifact outputs copied from the previous session>
|
|
||||||
```
|
|
||||||
|
|
||||||
The `previous/` directory is current-session state. It is a prepared input cache, not a full mirror of the previous session workspace.
|
|
||||||
|
|
||||||
Conceptually:
|
|
||||||
|
|
||||||
```text
|
|
||||||
work/<campaign>/<current_session_id>/
|
|
||||||
previous/
|
|
||||||
manifest.json
|
|
||||||
artifacts/
|
|
||||||
session_recap.md
|
|
||||||
quest_log.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The current session should not read directly from the previous session's local workspace during ordinary operation.
|
|
||||||
|
|
||||||
### 4. S3 is authoritative for previous-session state
|
|
||||||
|
|
||||||
For this initial implementation, previous-session artifacts should be downloaded from the configured S3 backend.
|
|
||||||
|
|
||||||
Do not compare local and remote copies.
|
|
||||||
|
|
||||||
Do not prefer local previous-session workspace state.
|
|
||||||
|
|
||||||
Do not implement a `--local` override in this roadmap. That can be considered later.
|
|
||||||
|
|
||||||
The simplified stage behavior is:
|
|
||||||
|
|
||||||
1. If the local manifest indicates `prepare` already succeeded and `--force` is not supplied, the runner skips `prepare`. No previous-session download occurs.
|
|
||||||
2. If `prepare` has not succeeded, `prepare` runs and downloads referenced previous-session artifacts from S3.
|
|
||||||
3. If `prepare` previously succeeded but `--force` is supplied, `prepare` runs again and overwrites local `previous/` state from S3.
|
|
||||||
|
|
||||||
### 5. Materialize only referenced previous-session artifacts
|
|
||||||
|
|
||||||
During `prepare`, scan the current resolved pipeline/session configuration for Scriptorium artifact inputs whose source matches:
|
|
||||||
|
|
||||||
```text
|
|
||||||
narratio.previous_session.artifact.<artifact_name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Only those referenced previous-session artifacts need to be downloaded.
|
|
||||||
|
|
||||||
Do not blindly download every previous-session artifact.
|
|
||||||
|
|
||||||
If no previous-session artifact sources are referenced, `prepare` should not require `previous_session_id` and should not touch `previous/`.
|
|
||||||
|
|
||||||
### 6. Preserve stage boundaries
|
|
||||||
|
|
||||||
`prepare` owns previous-session artifact materialization because these files are inputs to later stages.
|
|
||||||
|
|
||||||
`analyze` should not talk to S3.
|
|
||||||
|
|
||||||
The Scriptorium adapter should not know about previous sessions.
|
|
||||||
|
|
||||||
The storage adapter should not infer campaign, session, run, or root-prefix semantics. Callers should continue to provide explicit bucket-relative keys.
|
|
||||||
|
|
||||||
### 7. Archive and restore `previous/`
|
|
||||||
|
|
||||||
After implementation:
|
|
||||||
|
|
||||||
- `archive` should include `previous/` as durable current-session prepared input state.
|
|
||||||
- `restore` should restore `previous/` along with the rest of the durable session state it already restores.
|
|
||||||
- `previous/` should not be treated as current-session generated artifacts.
|
|
||||||
- `previous/` entries should be recorded as inputs/provenance, not as outputs produced by the current session.
|
|
||||||
|
|
||||||
## Target User Workflow
|
|
||||||
|
|
||||||
A typical session config:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
session_id: "{{ session_id }}"
|
|
||||||
previous_session_id: "{{ previous_session_id }}"
|
|
||||||
campaign: sample-campaign
|
|
||||||
|
|
||||||
inputs:
|
|
||||||
audio_dir: ./audio
|
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
A typical artifact config:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
scriptorium:
|
|
||||||
artifacts:
|
|
||||||
session_recap:
|
|
||||||
enabled: true
|
|
||||||
prompt_id: dnd_session.session_recap
|
|
||||||
output_path: artifacts/session_recap.md
|
|
||||||
inputs:
|
|
||||||
transcript:
|
|
||||||
source: narratio.transcript.trimmed
|
|
||||||
required: true
|
|
||||||
previous_recap:
|
|
||||||
source: narratio.previous_session.artifact.session_recap
|
|
||||||
required: false
|
|
||||||
```
|
|
||||||
|
|
||||||
Typical command:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run --session-id 2026-04-11 --previous-session-id 2026-04-04
|
|
||||||
```
|
|
||||||
|
|
||||||
Expected behavior:
|
|
||||||
|
|
||||||
1. `prepare` sees a referenced previous-session artifact: `session_recap`.
|
|
||||||
2. `prepare` downloads the previous session's `narratio.artifact.session_recap` from S3.
|
|
||||||
3. `prepare` writes it under the current session workspace, for example `previous/artifacts/session_recap.md`.
|
|
||||||
4. `prepare` records provenance in the current session manifest.
|
|
||||||
5. `analyze` resolves `narratio.previous_session.artifact.session_recap` from the local `previous/` directory.
|
|
||||||
6. Scriptorium receives the previous recap as a normal input file.
|
|
||||||
|
|
||||||
## Implementation Plan
|
|
||||||
|
|
||||||
### Phase 1: Session config and templating
|
|
||||||
|
|
||||||
Update session configuration structs to include:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
previous_session_id: ""
|
|
||||||
```
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
1. Add `PreviousSessionID` or equivalent to the session config type.
|
|
||||||
2. Add validation:
|
|
||||||
- optional;
|
|
||||||
- same format constraints as `session_id`;
|
|
||||||
- must not equal `session_id`.
|
|
||||||
3. Extend session templating support to include:
|
|
||||||
- `{{previous_session_id}}`
|
|
||||||
- `{{ previous_session_id }}`
|
|
||||||
4. Add a CLI flag if required by current templating flow:
|
|
||||||
- `--previous-session-id <id>`
|
|
||||||
5. Ensure unresolved `previous_session_id` placeholders fail clearly.
|
|
||||||
6. Update config tests for:
|
|
||||||
- no previous session;
|
|
||||||
- valid previous session;
|
|
||||||
- previous session equal to current session;
|
|
||||||
- unresolved placeholder;
|
|
||||||
- CLI/template rendering.
|
|
||||||
|
|
||||||
Do not add future workflow flags in this phase.
|
|
||||||
|
|
||||||
### Phase 2: Workspace path helpers
|
|
||||||
|
|
||||||
Add centralized path helpers for current-session previous-state paths.
|
|
||||||
|
|
||||||
Suggested helpers:
|
|
||||||
|
|
||||||
```text
|
|
||||||
SessionPreviousDir()
|
|
||||||
SessionPreviousManifestPath()
|
|
||||||
SessionPreviousArtifactsDir()
|
|
||||||
SessionPreviousArtifactPath(name or relative output path)
|
|
||||||
```
|
|
||||||
|
|
||||||
The exact names should match existing path-helper style.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- Do not construct `previous/` paths through scattered string concatenation.
|
|
||||||
- Keep paths session-relative where possible.
|
|
||||||
- Ensure workspace layout creation includes `previous/` only when appropriate, or creates it idempotently with the rest of the layout if simpler.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- path helper tests;
|
|
||||||
- workspace layout tests;
|
|
||||||
- ensure cleanup logic does not accidentally delete configured roots;
|
|
||||||
- ensure `previous/` is treated as session-durable state, not run-local state.
|
|
||||||
|
|
||||||
### Phase 3: Previous-session source parsing
|
|
||||||
|
|
||||||
Add parsing/recognition for:
|
|
||||||
|
|
||||||
```text
|
|
||||||
narratio.previous_session.artifact.<artifact_name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Implementation steps:
|
|
||||||
|
|
||||||
1. Add constants/helpers in the artifact/source parsing layer.
|
|
||||||
2. Validate artifact names using the same rules as current runtime artifact keys.
|
|
||||||
3. Add helpers such as:
|
|
||||||
- `IsPreviousSessionArtifactSource(source string) bool`
|
|
||||||
- `PreviousSessionArtifactName(source string) (string, bool)`
|
|
||||||
4. Ensure config validation accepts this source pattern.
|
|
||||||
5. Ensure invalid sources fail clearly.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- valid previous-session artifact source;
|
|
||||||
- invalid/missing artifact name;
|
|
||||||
- invalid artifact key characters;
|
|
||||||
- ordinary current-session sources still validate;
|
|
||||||
- unknown sources still fail.
|
|
||||||
|
|
||||||
### Phase 4: Scan configured artifacts for previous-session inputs
|
|
||||||
|
|
||||||
Add a helper that inspects resolved Scriptorium artifact definitions and returns the set of referenced previous-session artifact names.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- Scan enabled artifacts according to current artifact-enable semantics.
|
|
||||||
- Include all inputs whose source matches `narratio.previous_session.artifact.<name>`.
|
|
||||||
- Deduplicate artifact names.
|
|
||||||
- Sort results deterministically.
|
|
||||||
- Preserve required/optional information per reference.
|
|
||||||
- If the same previous artifact is referenced both required and optional, treat it as required.
|
|
||||||
|
|
||||||
Suggested output model:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type PreviousArtifactRequirement struct {
|
|
||||||
Name string
|
|
||||||
Required bool
|
|
||||||
Sources []string // optional diagnostics
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- no artifacts;
|
|
||||||
- no previous inputs;
|
|
||||||
- one optional previous input;
|
|
||||||
- one required previous input;
|
|
||||||
- duplicate references;
|
|
||||||
- required plus optional reference to the same artifact;
|
|
||||||
- deterministic ordering.
|
|
||||||
|
|
||||||
### Phase 5: Resolve previous-session archive keys
|
|
||||||
|
|
||||||
Implement a narrow service/helper used by `prepare` to resolve previous-session artifact files from S3.
|
|
||||||
|
|
||||||
Responsibilities:
|
|
||||||
|
|
||||||
1. Locate the previous session's current remote state.
|
|
||||||
2. Download the previous session manifest, or the minimum remote metadata needed to resolve artifact source IDs.
|
|
||||||
3. Resolve `narratio.artifact.<name>` inside the previous session's artifact catalog/manifest.
|
|
||||||
4. Download the resolved artifact into the current session's `previous/` directory.
|
|
||||||
5. Download/store the previous session manifest as `previous/manifest.json`.
|
|
||||||
6. Return provenance records for manifest input recording.
|
|
||||||
|
|
||||||
Important archive invariant:
|
|
||||||
|
|
||||||
- Remote current state must be based on the committed archive marker.
|
|
||||||
- Do not treat incomplete archive uploads as current state.
|
|
||||||
- Use the existing archive/current remote layout and commit-marker rules.
|
|
||||||
- `current/run_id.txt` is the final remote commit marker and should be respected when locating current remote state.
|
|
||||||
|
|
||||||
Do not put prefix semantics into the storage adapter. Compute explicit bucket-relative keys in app/stage/archive helper code, then call the storage adapter.
|
|
||||||
|
|
||||||
Required vs optional behavior:
|
|
||||||
|
|
||||||
- Required previous artifact missing from S3/current manifest: fail `prepare`.
|
|
||||||
- Optional previous artifact missing from S3/current manifest: continue without materializing that input.
|
|
||||||
- Previous session missing entirely:
|
|
||||||
- fail if any referenced previous artifact is required;
|
|
||||||
- continue if all referenced previous artifacts are optional.
|
|
||||||
- If previous_session_id is unset:
|
|
||||||
- fail if any referenced previous artifact is required;
|
|
||||||
- continue and omit all previous-session inputs if all are optional.
|
|
||||||
|
|
||||||
Validation behavior:
|
|
||||||
|
|
||||||
- Downloaded artifacts should pass the same validation rules as current-session artifacts where practical.
|
|
||||||
- Generic Scriptorium artifacts should at least be non-empty.
|
|
||||||
- Invalid required artifact: fail.
|
|
||||||
- Invalid optional artifact: prefer fail if the object exists but is invalid, because invalid archived data is usually an operator problem rather than absence.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- downloads previous manifest;
|
|
||||||
- downloads required previous artifact;
|
|
||||||
- skips missing optional previous artifact;
|
|
||||||
- fails missing required previous artifact;
|
|
||||||
- fails required previous artifact when previous_session_id is unset;
|
|
||||||
- optional previous artifact with no previous_session_id does not fail;
|
|
||||||
- respects current remote commit marker;
|
|
||||||
- does not use local previous-session workspace state;
|
|
||||||
- uses storage adapter with explicit keys.
|
|
||||||
|
|
||||||
### Phase 6: Integrate with `prepare`
|
|
||||||
|
|
||||||
Extend the `prepare` stage:
|
|
||||||
|
|
||||||
1. Run existing input materialization as before.
|
|
||||||
2. Detect previous-session artifact requirements.
|
|
||||||
3. If requirements exist, hydrate `previous/` from S3 according to the rules above.
|
|
||||||
4. Record hydrated previous artifacts in `manifest.Inputs`.
|
|
||||||
5. Preserve existing prepare outputs and provenance behavior.
|
|
||||||
|
|
||||||
Overwrite behavior:
|
|
||||||
|
|
||||||
- If `prepare` runs, it owns `previous/`.
|
|
||||||
- Before hydrating, clear the managed `previous/` directory, or clear the managed previous artifact paths.
|
|
||||||
- Prefer clearing the whole `previous/` directory if no other feature writes there.
|
|
||||||
- Under `--force`, this naturally overwrites local `previous/` state.
|
|
||||||
- Do not compare local and remote copies.
|
|
||||||
|
|
||||||
Skip behavior:
|
|
||||||
|
|
||||||
- Do not add stage-local skip logic.
|
|
||||||
- Runner-level skip remains authoritative.
|
|
||||||
- If the manifest says `prepare` succeeded and `--force` is not supplied, `prepare` does not run and no S3 downloads occur.
|
|
||||||
- If users add a new previous-session input after `prepare` already succeeded, they must rerun prepare with `--force`.
|
|
||||||
|
|
||||||
Error message requirement:
|
|
||||||
|
|
||||||
If an analyze-stage input cannot be resolved because `previous/` is missing or stale, the error should tell the operator to run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run-stage --force prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
or the appropriate existing CLI command shape.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- ordinary prepare without previous_session_id remains unchanged;
|
|
||||||
- prepare with optional previous artifact and no previous_session_id succeeds;
|
|
||||||
- prepare with required previous artifact and no previous_session_id fails;
|
|
||||||
- prepare with required previous artifact downloads to `previous/`;
|
|
||||||
- force prepare overwrites `previous/`;
|
|
||||||
- prepare records manifest inputs for previous artifacts;
|
|
||||||
- prepare skip behavior remains controlled by runner tests;
|
|
||||||
- no regression in S3 audio prepare behavior.
|
|
||||||
|
|
||||||
### Phase 7: Artifact resolver support
|
|
||||||
|
|
||||||
Update artifact resolution so Scriptorium inputs can resolve:
|
|
||||||
|
|
||||||
```text
|
|
||||||
narratio.previous_session.artifact.<artifact_name>
|
|
||||||
```
|
|
||||||
|
|
||||||
from the current session's `previous/` directory.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- The resolver should not call S3.
|
|
||||||
- The resolver should not read the previous session's local workspace.
|
|
||||||
- The resolver should map the previous-session source ID to the local prepared copy under `previous/`.
|
|
||||||
- Resolution should use manifest input records when available.
|
|
||||||
- Fallback to the local `previous/` path may be allowed if consistent with existing resolver behavior, but manifest provenance should be preferred.
|
|
||||||
- Missing required input should fail with a clear prepare-oriented message.
|
|
||||||
- Optional missing input should be omitted.
|
|
||||||
|
|
||||||
Suggested provenance:
|
|
||||||
|
|
||||||
```text
|
|
||||||
previous_session.manifest.outputs
|
|
||||||
previous_session.archive.current
|
|
||||||
current_session.previous_cache
|
|
||||||
```
|
|
||||||
|
|
||||||
Use names that fit the existing manifest/resolver vocabulary.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- resolves prepared previous artifact through manifest input record;
|
|
||||||
- resolves or fails appropriately when only filesystem copy exists, depending on chosen fallback policy;
|
|
||||||
- missing optional previous artifact is omitted;
|
|
||||||
- missing required previous artifact fails clearly;
|
|
||||||
- current-session `narratio.artifact.<name>` behavior is unchanged.
|
|
||||||
|
|
||||||
### Phase 8: Analyze-stage integration
|
|
||||||
|
|
||||||
The analyze stage should require little or no special previous-session logic if the resolver is designed correctly.
|
|
||||||
|
|
||||||
Confirm:
|
|
||||||
|
|
||||||
- Scriptorium input resolution accepts previous-session source IDs.
|
|
||||||
- The Scriptorium adapter receives a normal local input path.
|
|
||||||
- Render-debug and run modes behave the same as for ordinary inputs.
|
|
||||||
- Stage metadata includes useful input provenance if current structures support it.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- configured artifact receives previous recap input;
|
|
||||||
- optional previous recap omitted when not prepared;
|
|
||||||
- required previous recap fails when not prepared;
|
|
||||||
- render-debug path works with previous-session inputs;
|
|
||||||
- no S3 calls occur from analyze.
|
|
||||||
|
|
||||||
### Phase 9: Archive `previous/`
|
|
||||||
|
|
||||||
Update archive behavior so the current session's durable `previous/` directory is uploaded/preserved.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- `previous/` is current-session input/provenance state.
|
|
||||||
- It is not a generated current-session artifact.
|
|
||||||
- Archive it with the current session's durable state, alongside other durable session files according to the current archive layout.
|
|
||||||
- Preserve existing archive commit ordering.
|
|
||||||
- Do not make `previous/` upload the final commit marker.
|
|
||||||
- Do not treat missing `previous/` as an error when no previous-session inputs were prepared.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- archive includes `previous/manifest.json` and prepared previous artifacts when present;
|
|
||||||
- archive omits or tolerates absent `previous/` when unused;
|
|
||||||
- archive commit ordering remains valid;
|
|
||||||
- cleanup behavior does not delete durable `previous/` before archive commit.
|
|
||||||
|
|
||||||
### Phase 10: Restore `previous/`
|
|
||||||
|
|
||||||
Update `narratio restore` so it restores the current session's archived `previous/` directory.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- Restore `previous/` as durable current-session state.
|
|
||||||
- Do not infer or restore the previous session workspace merely because `previous_session_id` exists.
|
|
||||||
- Do not redownload previous-session artifacts from the previous session archive during restore; restore the current session's archived `previous/` cache.
|
|
||||||
- Respect existing restore flags and overwrite behavior.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- restore downloads `previous/` when present;
|
|
||||||
- restore succeeds when `previous/` is absent;
|
|
||||||
- restore with force overwrites local `previous/` according to existing restore semantics;
|
|
||||||
- restored current session can run analyze using `previous/` without needing previous session archive access.
|
|
||||||
|
|
||||||
### Phase 11: Documentation updates after implementation
|
|
||||||
|
|
||||||
After implementation, update current-behavior docs. Do not document implemented behavior only in this roadmap.
|
|
||||||
|
|
||||||
Likely files:
|
|
||||||
|
|
||||||
- `docs/config.md`
|
|
||||||
- `docs/cli.md`
|
|
||||||
- `docs/operations.md`
|
|
||||||
- `docs/internal/stage-prepare.md`
|
|
||||||
- `docs/internal/artifacts.md`
|
|
||||||
- `docs/internal/storage.md` only if storage contracts change
|
|
||||||
- `docs/internal/workspace.md`
|
|
||||||
- relevant maintained examples under `examples/`
|
|
||||||
|
|
||||||
Docs should explain:
|
|
||||||
|
|
||||||
- `session.previous_session_id`;
|
|
||||||
- `--previous-session-id` if added;
|
|
||||||
- `narratio.previous_session.artifact.<artifact_name>`;
|
|
||||||
- `previous/` workspace directory;
|
|
||||||
- S3-authoritative previous-session behavior;
|
|
||||||
- required vs optional previous-session artifact handling;
|
|
||||||
- when to run `narratio run-stage --force prepare`.
|
|
||||||
|
|
||||||
Do not document future `--local` support as current behavior.
|
|
||||||
|
|
||||||
## Future Work
|
|
||||||
|
|
||||||
These items are explicitly out of scope for the initial implementation.
|
|
||||||
|
|
||||||
### `--local` previous-session source
|
|
||||||
|
|
||||||
A future flag may allow `prepare` to copy previous-session artifacts from a local previous session workspace instead of S3.
|
|
||||||
|
|
||||||
Possible future command shape:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run-stage --force prepare --local
|
|
||||||
```
|
|
||||||
|
|
||||||
or a more specific flag such as:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run-stage --force prepare --previous-source local
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not implement this now.
|
|
||||||
|
|
||||||
### `narratio run --restore`
|
|
||||||
|
|
||||||
A future flag may run `narratio restore` before starting the regular pipeline:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run --restore --session-id 2026-04-11
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not implement this as part of previous-session artifact support unless it already exists and only requires documentation.
|
|
||||||
|
|
||||||
### Multi-previous-session support
|
|
||||||
|
|
||||||
A future design may support more than one previous/reference session.
|
|
||||||
|
|
||||||
Do not implement this now.
|
|
||||||
|
|
||||||
### Previous-session artifact version pinning
|
|
||||||
|
|
||||||
A future design may pin previous-session artifact inputs to a specific previous run ID or checksum.
|
|
||||||
|
|
||||||
Do not implement this now.
|
|
||||||
|
|
||||||
## Test Plan Summary
|
|
||||||
|
|
||||||
Run at least:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/config -v
|
|
||||||
go test ./internal/artifacts -v
|
|
||||||
go test ./internal/stage -run Prepare -v
|
|
||||||
go test ./internal/stage -run Analyze -v
|
|
||||||
go test ./internal/app -run TestExecute -v
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Add focused tests for:
|
|
||||||
|
|
||||||
- session config and templating;
|
|
||||||
- previous-session source parsing;
|
|
||||||
- previous artifact requirement scanning;
|
|
||||||
- S3-backed previous artifact download;
|
|
||||||
- prepare skip/force semantics;
|
|
||||||
- manifest input provenance;
|
|
||||||
- resolver behavior;
|
|
||||||
- analyze integration;
|
|
||||||
- archive/restore `previous/` persistence;
|
|
||||||
- examples load/validate.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
The feature is complete when:
|
|
||||||
|
|
||||||
1. `session.yml` supports optional `previous_session_id`.
|
|
||||||
2. Session templating supports `previous_session_id`.
|
|
||||||
3. Scriptorium artifact inputs accept `narratio.previous_session.artifact.<artifact_name>`.
|
|
||||||
4. `prepare` scans enabled Scriptorium artifacts for previous-session artifact inputs.
|
|
||||||
5. `prepare` downloads referenced previous-session artifacts from S3 into `previous/`.
|
|
||||||
6. Required/optional behavior is correct.
|
|
||||||
7. `prepare` uses stage-level skip/force behavior and does not compare local and remote copies.
|
|
||||||
8. `analyze` resolves previous-session artifacts only from local prepared `previous/` state.
|
|
||||||
9. `archive` persists `previous/`.
|
|
||||||
10. `restore` restores `previous/`.
|
|
||||||
11. Tests cover config, prepare, resolver, analyze, archive, and restore behavior.
|
|
||||||
12. Current-behavior docs and maintained examples are updated after implementation.
|
|
||||||
13. No storage adapter implementation infers campaign/session/root-prefix semantics.
|
|
||||||
14. No previous-session S3 logic is added to `analyze` or the Scriptorium adapter.
|
|
||||||
@@ -9,19 +9,19 @@ Symptom:
|
|||||||
- `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found.
|
- `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found.
|
||||||
|
|
||||||
Likely Cause:
|
Likely Cause:
|
||||||
- `pipeline.yml` or `session.yml` is missing from discovery paths.
|
- `pipeline.yml`, `campaign.yml`, or `session.yml` is missing from system discovery paths.
|
||||||
- wrong working directory when relying on `./session.yml`.
|
- a local working-directory config file was not passed explicitly.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pwd
|
|
||||||
ls -l ./session.yml
|
|
||||||
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
|
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
|
||||||
|
ls -l /usr/local/etc/narratio/campaign.yml /etc/narratio/campaign.yml
|
||||||
|
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
- pass explicit `--config` and `--session`.
|
- pass explicit `--config`, `--campaign`, and `--session`.
|
||||||
- or place files in documented discovery paths.
|
- or place files in documented discovery paths.
|
||||||
|
|
||||||
Links:
|
Links:
|
||||||
@@ -40,7 +40,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio plan --session ./session.yml --session-id 2026-04-04
|
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session ./session.yml --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -62,7 +62,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
|
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -84,7 +84,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
|
narratio run --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -106,7 +106,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
|
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -129,7 +129,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
|
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -154,7 +154,7 @@ Diagnostics:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio status --manifest /path/to/manifest.json
|
narratio status --manifest /path/to/manifest.json
|
||||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
|
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -227,7 +227,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -250,7 +250,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -272,7 +272,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -342,7 +342,7 @@ Likely Cause:
|
|||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
|
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
@@ -366,7 +366,7 @@ Diagnostics:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio status --manifest /path/to/manifest.json
|
narratio status --manifest /path/to/manifest.json
|
||||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 archive
|
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 archive
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe Fix:
|
||||||
|
|||||||
5
examples/campaign.yml
Normal file
5
examples/campaign.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
@@ -1,9 +1,5 @@
|
|||||||
session_id: 2026-05-03
|
session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
date: 2026-05-03
|
date: 2026-05-03
|
||||||
title: Sample Session
|
title: Sample Session
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
session_id: 2026-05-03
|
session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
date: 2026-05-03
|
date: 2026-05-03
|
||||||
title: Sample Session
|
title: Sample Session
|
||||||
inputs:
|
inputs:
|
||||||
audio_s3:
|
audio_s3:
|
||||||
prefix: audio/
|
prefix: audio/
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -1,7 +1,3 @@
|
|||||||
session_id: "{{ session_id }}"
|
session_id: "{{ session_id }}"
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -27,8 +27,9 @@ type FakeBackend struct {
|
|||||||
Err error
|
Err error
|
||||||
Result ArchiveResult
|
Result ArchiveResult
|
||||||
|
|
||||||
Objects map[string]FakeObject
|
Objects map[string]FakeObject
|
||||||
Uploads []FakeUploadCall
|
Uploads []FakeUploadCall
|
||||||
|
Downloads []FakeDownloadCall
|
||||||
|
|
||||||
ListErr error
|
ListErr error
|
||||||
DownloadErr error
|
DownloadErr error
|
||||||
@@ -43,6 +44,12 @@ type FakeUploadCall struct {
|
|||||||
Options UploadOptions
|
Options UploadOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FakeDownloadCall captures one download invocation in call order.
|
||||||
|
type FakeDownloadCall struct {
|
||||||
|
Key string
|
||||||
|
LocalPath string
|
||||||
|
}
|
||||||
|
|
||||||
// Archive records request and returns configured response.
|
// Archive records request and returns configured response.
|
||||||
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -130,6 +137,10 @@ func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error
|
|||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||||
}
|
}
|
||||||
|
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||||
|
Key: normalizeObjectKey(key),
|
||||||
|
LocalPath: localPath,
|
||||||
|
})
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ func TestFakeBackendDownload(t *testing.T) {
|
|||||||
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||||
|
|
||||||
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||||
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
|
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||||
t.Fatalf("Download() error = %v", err)
|
t.Fatalf("Download() error = %v", err)
|
||||||
}
|
}
|
||||||
data, err := os.ReadFile(dst)
|
data, err := os.ReadFile(dst)
|
||||||
|
|||||||
@@ -9,17 +9,19 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
[]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -33,12 +35,12 @@ func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
[]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -52,7 +54,7 @@ func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -65,7 +67,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(
|
err := RunStage(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
||||||
&out,
|
&out,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -78,7 +80,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
|||||||
|
|
||||||
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -93,7 +95,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(
|
err := Resume(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap"},
|
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
&out,
|
&out,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -104,10 +106,154 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string) {
|
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedStages []string
|
||||||
|
var capturedForce bool
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
for _, s := range stages {
|
||||||
|
capturedStages = append(capturedStages, s.Name())
|
||||||
|
}
|
||||||
|
capturedForce = opts.Force
|
||||||
|
return &RunSummary{
|
||||||
|
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||||
|
Executed: []string{"analyze"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
|
||||||
|
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
|
||||||
|
}
|
||||||
|
if !capturedForce {
|
||||||
|
t.Fatal("captured force = false, want true")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
|
||||||
|
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedArtifacts []string
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
|
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{
|
||||||
|
"analyze",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--artifacts", "player_handout,session_recap",
|
||||||
|
},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
|
||||||
|
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "positional", args: []string{"analyze", "extra"}, want: "analyze: unexpected positional arguments"},
|
||||||
|
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(tc.args, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), tc.want) {
|
||||||
|
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"analyze"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||||
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteUsageIncludesAnalyze(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(nil, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "analyze") {
|
||||||
|
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open pipeline config for append: %v", err)
|
t.Fatalf("open pipeline config for append: %v", err)
|
||||||
@@ -136,5 +282,5 @@ scriptorium:
|
|||||||
if _, err := f.WriteString(extra); err != nil {
|
if _, err := f.WriteString(extra); err != nil {
|
||||||
t.Fatalf("append scriptorium config: %v", err)
|
t.Fatalf("append scriptorium config: %v", err)
|
||||||
}
|
}
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
}
|
}
|
||||||
|
|||||||
49
internal/app/campaign_config_path.go
Normal file
49
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolveCampaignConfigPath(flagValue string) (string, error) {
|
||||||
|
return resolveCampaignConfigPathWithCandidates(flagValue, config.DefaultCampaignConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveCampaignConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||||
|
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||||
|
return explicit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ordered := make([]string, 0, len(candidates))
|
||||||
|
for _, raw := range candidates {
|
||||||
|
path := strings.TrimSpace(raw)
|
||||||
|
if path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ordered = append(ordered, path)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
if info.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("check default campaign config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ordered) == 0 {
|
||||||
|
return "", fmt.Errorf("no campaign config path provided and no default locations configured")
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"no campaign config path provided and no default campaign config found; searched: %s; pass --campaign to use an explicit path",
|
||||||
|
strings.Join(ordered, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
49
internal/app/campaign_config_path_test.go
Normal file
49
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathExplicitWins(t *testing.T) {
|
||||||
|
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||||
|
got, err := resolveCampaignConfigPathWithCandidates(explicit, []string{filepath.Join(t.TempDir(), "campaign.yml")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != explicit {
|
||||||
|
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathUsesFirstExistingDefault(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
missing := filepath.Join(dir, "missing.yml")
|
||||||
|
found := filepath.Join(dir, "campaign.yml")
|
||||||
|
if err := os.WriteFile(found, []byte("campaign: sample-campaign\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolveCampaignConfigPathWithCandidates("", []string{missing, found})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(found) {
|
||||||
|
t.Fatalf("path = %q, want %q", got, filepath.Clean(found))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||||
|
_, err := resolveCampaignConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/campaign.yml", "/etc/narratio/campaign.yml"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "searched") {
|
||||||
|
t.Fatalf("error = %q, want searched paths", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "pass --campaign") {
|
||||||
|
t.Fatalf("error = %q, want explicit-campaign guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
336
internal/app/clean.go
Normal file
336
internal/app/clean.go
Normal file
@@ -0,0 +1,336 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Clean removes local workspace/spool state while preserving durable cache
|
||||||
|
// state unless cache cleanup is explicitly requested.
|
||||||
|
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var all bool
|
||||||
|
var dryRun bool
|
||||||
|
var clearCache bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
||||||
|
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
||||||
|
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("clean: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("clean: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
if all {
|
||||||
|
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||||
|
}
|
||||||
|
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("clean: --session-id is required unless --all is set")
|
||||||
|
}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
|
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||||
|
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||||
|
if campaign == "" || sessionID == "" {
|
||||||
|
return fmt.Errorf("clean: campaign and session_id are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
||||||
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if clearCache {
|
||||||
|
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cache: preserved")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||||
|
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||||
|
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||||
|
strings.TrimSpace(flags.sessionID) != "" ||
|
||||||
|
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||||
|
return fmt.Errorf("clean: --all cannot be combined with --campaign, --session, --session-id, or --previous-session-id")
|
||||||
|
}
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintln(out, "Clean plan for all local sessions")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cleaned all local sessions")
|
||||||
|
}
|
||||||
|
|
||||||
|
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
||||||
|
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if clearCache {
|
||||||
|
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cache: preserved")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
||||||
|
dir, err := validateScopedDir(root, target, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
if dir.Exists {
|
||||||
|
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !dir.Exists {
|
||||||
|
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
||||||
|
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(entry); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
|
}
|
||||||
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(rootAbs)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return rootAbs, nil, nil
|
||||||
|
}
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(rootAbs)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
path := filepath.Join(rootAbs, entry.Name())
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
||||||
|
}
|
||||||
|
out = append(out, path)
|
||||||
|
}
|
||||||
|
return rootAbs, out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||||
|
if cfg.Session.Inputs.AudioS3 == nil {
|
||||||
|
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||||
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||||
|
objects, err := store.List(ctx, audioPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||||
|
}
|
||||||
|
count := 0
|
||||||
|
for _, obj := range objects {
|
||||||
|
key := strings.TrimSpace(obj.Key)
|
||||||
|
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if count == 0 {
|
||||||
|
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
||||||
|
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
||||||
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||||
|
}
|
||||||
|
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
||||||
|
file, err := validateScopedFile(root, target, policy)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
if file.Exists {
|
||||||
|
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if !file.Exists {
|
||||||
|
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := os.Remove(file.TargetAbs); err != nil {
|
||||||
|
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
cleanTarget := strings.TrimSpace(target)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
|
}
|
||||||
|
if cleanTarget == "" {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||||
|
}
|
||||||
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
|
}
|
||||||
|
targetAbs, err := filepath.Abs(cleanTarget)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||||
|
}
|
||||||
|
if rel == "." {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||||
|
}
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
|
||||||
|
}
|
||||||
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanIsFlac(path string) bool {
|
||||||
|
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||||
|
}
|
||||||
255
internal/app/clean_test.go
Normal file
255
internal/app/clean_test.go
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, workDir)
|
||||||
|
cleanAssertMissing(t, spoolDir)
|
||||||
|
cleanAssertExists(t, cachePath)
|
||||||
|
if !strings.Contains(stdout.String(), "Cache: preserved") {
|
||||||
|
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--dry-run"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertExists(t, workDir)
|
||||||
|
cleanAssertExists(t, spoolDir)
|
||||||
|
if !strings.Contains(stdout.String(), "Would delete:") {
|
||||||
|
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Missing:") {
|
||||||
|
t.Fatalf("stdout = %q, want missing path output", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||||
|
mustWriteTestFile(t, otherCachePath, "other-audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, cachePath)
|
||||||
|
cleanAssertExists(t, otherCachePath)
|
||||||
|
if storeInitCalls != 1 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
|
||||||
|
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work")
|
||||||
|
spoolRoot := filepath.Join(workspaceRoot, "spool")
|
||||||
|
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
mustWriteTestFile(t, cachePath, "cache")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, workRoot)
|
||||||
|
cleanAssertExists(t, spoolRoot)
|
||||||
|
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
|
||||||
|
cleanAssertExists(t, cachePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||||
|
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, audioCachePath, "cached-audio")
|
||||||
|
mustWriteTestFile(t, otherCachePath, "other-cache")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, audioCachePath)
|
||||||
|
cleanAssertExists(t, otherCachePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--all"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "--all cannot be combined") {
|
||||||
|
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanRequiresSessionID(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "--session-id is required unless --all is set") {
|
||||||
|
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanRejectsUnsafeTargets(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
|
||||||
|
t.Fatal("outside target error = nil, want error")
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
|
||||||
|
t.Fatal("root target error = nil, want error")
|
||||||
|
}
|
||||||
|
filePath := filepath.Join(root, "file.txt")
|
||||||
|
mustWriteTestFile(t, filePath, "file")
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
|
||||||
|
t.Fatal("file target error = nil, want error")
|
||||||
|
}
|
||||||
|
symlinkPath := filepath.Join(root, "link")
|
||||||
|
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
|
||||||
|
t.Fatalf("Symlink() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
|
||||||
|
t.Fatal("symlink target error = nil, want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClearIsNotCommandAlias(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clear"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAssertExists(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("expected %q to exist: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAssertMissing(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "restore"}
|
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "analyze", "restore", "session", "artifacts", "locks", "clean"}
|
||||||
|
|
||||||
// Execute dispatches CLI commands and returns a process exit code.
|
// Execute dispatches CLI commands and returns a process exit code.
|
||||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||||
@@ -32,8 +32,18 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
|||||||
err = Resume(ctx, cmdArgs, stdout)
|
err = Resume(ctx, cmdArgs, stdout)
|
||||||
case "run-stage":
|
case "run-stage":
|
||||||
err = RunStage(ctx, cmdArgs, stdout)
|
err = RunStage(ctx, cmdArgs, stdout)
|
||||||
|
case "analyze":
|
||||||
|
err = Analyze(ctx, cmdArgs, stdout)
|
||||||
case "restore":
|
case "restore":
|
||||||
err = Restore(ctx, cmdArgs, stdout)
|
err = Restore(ctx, cmdArgs, stdout)
|
||||||
|
case "session":
|
||||||
|
err = Session(ctx, cmdArgs, stdout)
|
||||||
|
case "artifacts":
|
||||||
|
err = Artifacts(ctx, cmdArgs, stdout)
|
||||||
|
case "locks":
|
||||||
|
err = Locks(ctx, cmdArgs, stdout)
|
||||||
|
case "clean":
|
||||||
|
err = Clean(ctx, cmdArgs, stdout)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||||
printUsage(stderr)
|
printUsage(stderr)
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := writeManifestPathForExecute(t)
|
manifestPath := writeManifestPathForExecute(t)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -32,11 +32,11 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -94,12 +94,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -110,14 +110,14 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -136,19 +136,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
stdout.Reset()
|
stdout.Reset()
|
||||||
stderr.Reset()
|
stderr.Reset()
|
||||||
|
|
||||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
code = Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -188,6 +188,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
|||||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||||
|
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
@@ -239,7 +240,7 @@ inputs:
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -252,6 +253,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -288,7 +290,7 @@ inputs:
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -305,11 +307,14 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||||
|
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||||
|
config.DefaultCampaignConfigSearchPaths = []string{campaignPath}
|
||||||
defer func() {
|
defer func() {
|
||||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||||
|
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||||
}()
|
}()
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
@@ -323,6 +328,33 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteMissingCampaignConfigReportsSearchedPaths(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
missingCampaignPath := filepath.Join(t.TempDir(), "campaign.yml")
|
||||||
|
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||||
|
config.DefaultCampaignConfigSearchPaths = []string{missingCampaignPath}
|
||||||
|
defer func() {
|
||||||
|
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||||
|
}()
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "no campaign config path provided and no default campaign config found; searched:") {
|
||||||
|
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "pass --campaign") {
|
||||||
|
t.Fatalf("stderr = %q, want explicit campaign guidance", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteInvalidCommand(t *testing.T) {
|
func TestExecuteInvalidCommand(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -359,11 +391,12 @@ func TestExecuteMissingCommand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
url := "https://example.com/transcribe"
|
url := "https://example.com/transcribe"
|
||||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||||
@@ -377,6 +410,10 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
|||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
|
cache:
|
||||||
|
root: ` + filepath.Join(workspaceRoot, "cache") + `
|
||||||
|
spool:
|
||||||
|
root: ` + filepath.Join(workspaceRoot, "spool") + `
|
||||||
storage:
|
storage:
|
||||||
backend: s3
|
backend: s3
|
||||||
s3:
|
s3:
|
||||||
@@ -407,9 +444,11 @@ notification:
|
|||||||
`
|
`
|
||||||
|
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
|
`
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
@@ -418,6 +457,9 @@ inputs:
|
|||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline config: %v", err)
|
t.Fatalf("write pipeline config: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign config: %v", err)
|
||||||
|
}
|
||||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write session config: %v", err)
|
t.Fatalf("write session config: %v", err)
|
||||||
}
|
}
|
||||||
@@ -427,7 +469,22 @@ inputs:
|
|||||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||||
|
t.Helper()
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
return campaignPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeManifestPathForExecute(t *testing.T) string {
|
func writeManifestPathForExecute(t *testing.T) string {
|
||||||
|
|||||||
133
internal/app/config_loader.go
Normal file
133
internal/app/config_loader.go
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvedCampaignPath, err := resolveCampaignConfigPath(campaignFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||||
|
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, explicitSession, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if discoveredSession.Path != "" {
|
||||||
|
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, discoveredSession.Path, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires --session-id")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
|
||||||
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
partialCfg := &config.Config{
|
||||||
|
Pipeline: pipelineCfg,
|
||||||
|
Campaign: campaignCfg,
|
||||||
|
PipelinePath: resolvedPipelinePath,
|
||||||
|
CampaignPath: resolvedCampaignPath,
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||||
|
}
|
||||||
|
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||||
|
}
|
||||||
|
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(pipelineCfg)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.Resolve(
|
||||||
|
resolvedPipelinePath,
|
||||||
|
pipelineCfg,
|
||||||
|
resolvedCampaignPath,
|
||||||
|
campaignCfg,
|
||||||
|
sessionTempPath,
|
||||||
|
sessionCfg,
|
||||||
|
config.SessionSource{
|
||||||
|
Source: "session_config.s3",
|
||||||
|
LocalPath: sessionTempPath,
|
||||||
|
S3Bucket: s3BucketName(pipelineCfg),
|
||||||
|
S3Key: remoteKey,
|
||||||
|
S3Size: sessionInfo.Size,
|
||||||
|
S3ETag: sessionInfo.ETag,
|
||||||
|
SpoolPath: sessionTempPath,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||||
|
objects, err := store.List(ctx, sessionPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||||
|
}
|
||||||
|
for _, obj := range objects {
|
||||||
|
if obj.Key == remoteKey {
|
||||||
|
return obj, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
|
||||||
|
f, err := os.CreateTemp("", "narratio-session-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := f.Name()
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("close temp file %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := store.Download(ctx, remoteKey, path); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||||
|
if cfg == nil || cfg.Storage.S3 == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||||
|
}
|
||||||
21
internal/app/object_store.go
Normal file
21
internal/app/object_store.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||||
|
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||||
|
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||||
|
}
|
||||||
|
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||||
|
}
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
165
internal/app/object_store_test.go
Normal file
165
internal/app/object_store_test.go
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||||
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
|
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||||
|
|
||||||
|
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
called := false
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
called = true
|
||||||
|
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||||
|
return nil, errors.New("access key was not loaded before object store init")
|
||||||
|
}
|
||||||
|
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||||
|
return nil, errors.New("secret key was not loaded before object store init")
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||||
|
}
|
||||||
|
if store != fake {
|
||||||
|
t.Fatalf("store = %#v, want fake backend", store)
|
||||||
|
}
|
||||||
|
if !called {
|
||||||
|
t.Fatal("object store factory was not called")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||||
|
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||||
|
t.Setenv(secretKeyEnv, "existing-secret")
|
||||||
|
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||||
|
|
||||||
|
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||||
|
return nil, errors.New("existing access key was overwritten")
|
||||||
|
}
|
||||||
|
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||||
|
return nil, errors.New("existing secret key was overwritten")
|
||||||
|
}
|
||||||
|
return &storage.FakeBackend{}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||||
|
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||||
|
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||||
|
called := false
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
called = true
|
||||||
|
return &storage.FakeBackend{}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("object store factory was called after secret load failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||||
|
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||||
|
cfg := commandObjectStoreTestConfig("")
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
return nil, errors.New("factory boom")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
|
||||||
|
t.Fatalf("error = %q, want factory context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
Backend: "s3",
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||||
|
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(secretsDir) != "" {
|
||||||
|
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||||
|
t.Helper()
|
||||||
|
originals := make(map[string]string, len(names))
|
||||||
|
present := make(map[string]bool, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
value, ok := os.LookupEnv(name)
|
||||||
|
originals[name] = value
|
||||||
|
present[name] = ok
|
||||||
|
_ = os.Unsetenv(name)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, name := range names {
|
||||||
|
if present[name] {
|
||||||
|
_ = os.Setenv(name, originals[name])
|
||||||
|
} else {
|
||||||
|
_ = os.Unsetenv(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
958
internal/app/operator_helpers.go
Normal file
958
internal/app/operator_helpers.go
Normal file
@@ -0,0 +1,958 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type commonConfigFlags struct {
|
||||||
|
pipelinePath string
|
||||||
|
campaignPath string
|
||||||
|
sessionPath string
|
||||||
|
sessionID string
|
||||||
|
previousSessionID string
|
||||||
|
}
|
||||||
|
|
||||||
|
type finding struct {
|
||||||
|
Severity string
|
||||||
|
Category string
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
type findingError struct {
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e findingError) Error() string {
|
||||||
|
return fmt.Sprintf("%d validation error(s)", e.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||||
|
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&flags.campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||||
|
return config.SessionLoadOptions{
|
||||||
|
SessionID: f.sessionID,
|
||||||
|
PreviousSessionID: f.previousSessionID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Session dispatches session helper subcommands.
|
||||||
|
func Session(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("session: expected subcommand: validate|init")
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "validate":
|
||||||
|
return SessionValidate(ctx, args[1:], out)
|
||||||
|
case "init":
|
||||||
|
return SessionInit(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("session: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Artifacts dispatches artifact helper subcommands.
|
||||||
|
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("artifacts: expected subcommand: list")
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "list":
|
||||||
|
return ArtifactsList(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionValidate performs a read-only session preflight.
|
||||||
|
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("session validate: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("session validate: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
|
||||||
|
findings := []finding{}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
|
return renderFindings(out, "", "", findings)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
|
} else {
|
||||||
|
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
|
||||||
|
}
|
||||||
|
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
|
||||||
|
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
findings = append(findings, validateStableInputFindings(cfg)...)
|
||||||
|
findings = append(findings, validateLocalAudioFindings(cfg)...)
|
||||||
|
|
||||||
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
|
if storeErr != nil {
|
||||||
|
findings = append(findings, errorFinding("storage", storeErr.Error()))
|
||||||
|
}
|
||||||
|
if cfg.Session.Inputs.AudioS3 != nil {
|
||||||
|
if storeErr != nil {
|
||||||
|
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
|
||||||
|
} else {
|
||||||
|
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||||
|
if len(requirements) == 0 {
|
||||||
|
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||||
|
} else if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
||||||
|
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
||||||
|
} else if storeErr != nil {
|
||||||
|
findings = append(findings, errorFinding("previous", "previous-session artifacts cannot be checked because storage is unavailable"))
|
||||||
|
} else {
|
||||||
|
findings = append(findings, validatePreviousArtifactFindings(ctx, cfg, store, requirements)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
locks, lockErr := loadEffectiveLocks(ctx, cfg, store)
|
||||||
|
if lockErr != nil {
|
||||||
|
findings = append(findings, errorFinding("locks", lockErr.Error()))
|
||||||
|
} else if len(locks.All) == 0 {
|
||||||
|
findings = append(findings, okFinding("locks", "no effective archive locks"))
|
||||||
|
} else {
|
||||||
|
for _, lock := range locks.All {
|
||||||
|
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if paths.ManifestPath != "" {
|
||||||
|
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
|
||||||
|
}
|
||||||
|
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status reports either a requested manifest or effective local/remote session state.
|
||||||
|
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var manifestPath string
|
||||||
|
var flags commonConfigFlags
|
||||||
|
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("status: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("status: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(manifestPath) != "" {
|
||||||
|
return statusManifest(ctx, manifestPath, out)
|
||||||
|
}
|
||||||
|
if flags.pipelinePath == "" && flags.campaignPath == "" && flags.sessionPath == "" && flags.sessionID == "" && flags.previousSessionID == "" {
|
||||||
|
return fmt.Errorf("status: --manifest is required")
|
||||||
|
}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("status: %w", err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return fmt.Errorf("status: %w", err)
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
|
||||||
|
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
|
||||||
|
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
|
||||||
|
|
||||||
|
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||||
|
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||||
|
} else if m == nil {
|
||||||
|
fmt.Fprintln(out, "Local manifest: missing")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||||
|
writeStageStatuses(out, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
|
if storeErr != nil {
|
||||||
|
fmt.Fprintf(out, "Remote archive: unavailable: %v\n", storeErr)
|
||||||
|
} else if store != nil {
|
||||||
|
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(out, "Remote archive: missing or unavailable: %v\n", err)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Remote archive: current run %s\n", current.RunID)
|
||||||
|
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||||
|
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||||
|
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||||
|
} else if storeErr == nil {
|
||||||
|
catalogLocks := locks
|
||||||
|
if err != nil {
|
||||||
|
catalogLocks = &effectiveLocks{
|
||||||
|
Static: staticArchiveLocks(cfg),
|
||||||
|
All: staticArchiveLocks(cfg),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
promotedRemoteState := map[string]string{}
|
||||||
|
if store != nil {
|
||||||
|
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Remote outputs:")
|
||||||
|
writeArtifactList(out, cfg, catalog, catalogLocks, promotedRemoteState)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(out, "Archive locks: error: %v\n", err)
|
||||||
|
} else {
|
||||||
|
writeLocks(out, cfg, locks)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Next actions:")
|
||||||
|
fmt.Fprintf(out, "- narratio session validate --session-id %s\n", cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "- narratio restore --session-id %s --dry-run\n", cfg.Session.SessionID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func statusManifest(ctx context.Context, manifestPath string, out io.Writer) error {
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
m, err := store.Load(ctx, manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("status: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
writeStageStatuses(out, m)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionInit creates a local or remote session.yml skeleton.
|
||||||
|
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var pipelinePath, campaignPath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||||
|
var remote, force bool
|
||||||
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
|
||||||
|
fs.StringVar(&date, "date", "", "session date")
|
||||||
|
fs.StringVar(&title, "title", "", "session title")
|
||||||
|
fs.StringVar(&output, "output", "", "local output session.yml path")
|
||||||
|
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
|
||||||
|
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||||
|
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||||
|
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("session init: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("session init: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(pipelinePath) == "" || strings.TrimSpace(campaignPath) == "" || strings.TrimSpace(sessionID) == "" {
|
||||||
|
return fmt.Errorf("session init: --config, --campaign, and --session-id are required")
|
||||||
|
}
|
||||||
|
if (strings.TrimSpace(output) == "") == !remote {
|
||||||
|
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
|
||||||
|
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedPipeline, err := resolvePipelineConfigPath(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
resolvedCampaign, err := resolveCampaignConfigPath(campaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipeline)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
campaignCfg, err := config.LoadCampaign(resolvedCampaign)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := buildSessionYAML(campaignCfg.Campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
label := strings.TrimSpace(output)
|
||||||
|
if label == "" {
|
||||||
|
label = "remote session.yml"
|
||||||
|
}
|
||||||
|
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
cfg, err := config.Resolve(resolvedPipeline, pipelineCfg, resolvedCampaign, campaignCfg, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !remote {
|
||||||
|
if err := writeLocalFile(output, data, force); err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
|
||||||
|
key := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
exists, err := store.Exists(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: check remote session %q: %w", key, err)
|
||||||
|
}
|
||||||
|
if exists && !force {
|
||||||
|
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
defer func() { _ = os.Remove(tmpPath) }()
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("session init: write temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("session init: close temp file: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||||
|
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(pipelineCfg), key)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArtifactsList lists effective artifact sources.
|
||||||
|
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var remote bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.BoolVar(&remote, "remote", false, "inspect remote archive availability")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("artifacts list: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("artifacts list: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
|
}
|
||||||
|
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
|
}
|
||||||
|
promotedRemoteState := map[string]string{}
|
||||||
|
if remote && store != nil {
|
||||||
|
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
||||||
|
}
|
||||||
|
writeArtifactList(out, cfg, catalog, locks, promotedRemoteState)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Locks dispatches archive lock list and mutation helpers.
|
||||||
|
func Locks(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
switch args[0] {
|
||||||
|
case "add":
|
||||||
|
return LocksAdd(ctx, args[1:], out)
|
||||||
|
case "remove":
|
||||||
|
return LocksRemove(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("locks: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return LocksList(ctx, args, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksList lists effective archive locks.
|
||||||
|
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("locks: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("locks: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks: --session-id is required")
|
||||||
|
}
|
||||||
|
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks: %w", err)
|
||||||
|
}
|
||||||
|
writeLocks(out, cfg, locks)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksAdd adds or updates one remote lock.
|
||||||
|
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var reason string
|
||||||
|
var force bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.StringVar(&reason, "reason", "", "lock reason")
|
||||||
|
fs.BoolVar(&force, "force", false, "update existing remote lock")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("locks add: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 1 {
|
||||||
|
return fmt.Errorf("locks add: expected exactly one source id")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks add: --session-id is required")
|
||||||
|
}
|
||||||
|
source := strings.TrimSpace(fs.Arg(0))
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||||
|
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||||
|
}
|
||||||
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
|
if _, exists := remoteSet[source]; exists && !force {
|
||||||
|
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||||
|
}
|
||||||
|
remoteSet[source] = config.ArchiveLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||||
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
|
if _, err := config.ValidateArchiveLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio locks add: locked %s\n", source)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksRemove removes one remote lock.
|
||||||
|
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("locks remove: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 1 {
|
||||||
|
return fmt.Errorf("locks remove: expected exactly one source id")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks remove: --session-id is required")
|
||||||
|
}
|
||||||
|
source := strings.TrimSpace(fs.Arg(0))
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
|
if _, ok := remoteSet[source]; !ok {
|
||||||
|
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||||
|
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
||||||
|
}
|
||||||
|
delete(remoteSet, source)
|
||||||
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio locks remove: unlocked %s\n", source)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
var store storage.ObjectStore
|
||||||
|
if needStore {
|
||||||
|
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||||
|
}
|
||||||
|
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
return cfg, store, locks, m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
|
||||||
|
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
|
||||||
|
date = strings.TrimSpace(sessionID)
|
||||||
|
}
|
||||||
|
type audioS3 struct {
|
||||||
|
Prefix string `yaml:"prefix"`
|
||||||
|
}
|
||||||
|
type inputs struct {
|
||||||
|
AudioDir string `yaml:"audio_dir,omitempty"`
|
||||||
|
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
|
||||||
|
}
|
||||||
|
type sessionYAML struct {
|
||||||
|
Campaign string `yaml:"campaign"`
|
||||||
|
SessionID string `yaml:"session_id"`
|
||||||
|
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
|
||||||
|
Date string `yaml:"date,omitempty"`
|
||||||
|
Title string `yaml:"title,omitempty"`
|
||||||
|
Inputs inputs `yaml:"inputs"`
|
||||||
|
}
|
||||||
|
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
|
||||||
|
if in.AudioDir == "" {
|
||||||
|
prefix := strings.TrimSpace(audioS3Prefix)
|
||||||
|
if prefix == "" {
|
||||||
|
prefix = "audio/"
|
||||||
|
}
|
||||||
|
in.AudioS3 = &audioS3{Prefix: prefix}
|
||||||
|
}
|
||||||
|
data, err := yaml.Marshal(sessionYAML{
|
||||||
|
Campaign: strings.TrimSpace(campaign),
|
||||||
|
SessionID: strings.TrimSpace(sessionID),
|
||||||
|
PreviousSessionID: strings.TrimSpace(previousSessionID),
|
||||||
|
Date: strings.TrimSpace(date),
|
||||||
|
Title: strings.TrimSpace(title),
|
||||||
|
Inputs: in,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
|
||||||
|
if campaign != "" || sessionID != "" {
|
||||||
|
fmt.Fprintf(out, "Campaign: %s\n", campaign)
|
||||||
|
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
|
||||||
|
}
|
||||||
|
errorsCount := 0
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.Severity == "ERROR" {
|
||||||
|
errorsCount++
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
|
||||||
|
}
|
||||||
|
if errorsCount > 0 {
|
||||||
|
return findingError{count: errorsCount}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
|
||||||
|
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
|
||||||
|
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
|
||||||
|
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
|
||||||
|
|
||||||
|
func sessionSourceSummary(cfg *config.Config) string {
|
||||||
|
source := cfg.SessionSource.Source
|
||||||
|
if source == "" {
|
||||||
|
source = "session_config"
|
||||||
|
}
|
||||||
|
if cfg.SessionSource.S3Key != "" {
|
||||||
|
return source + " " + cfg.SessionSource.S3Key
|
||||||
|
}
|
||||||
|
return source + " " + cfg.SessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStableInputFindings(cfg *config.Config) []finding {
|
||||||
|
items := []struct {
|
||||||
|
name string
|
||||||
|
in config.ResolvedInputFile
|
||||||
|
}{
|
||||||
|
{"speakers", cfg.StableInputs.SpeakersFile},
|
||||||
|
{"autocorrect", cfg.StableInputs.AutocorrectFile},
|
||||||
|
{"glossary", cfg.StableInputs.GlossaryFile},
|
||||||
|
}
|
||||||
|
out := make([]finding, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
path, err := resolveHelperConfigRelativePath(item.in)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, errorFinding("inputs", item.name+": "+err.Error()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
out = append(out, errorFinding("inputs", fmt.Sprintf("%s missing: %v", item.name, err)))
|
||||||
|
} else {
|
||||||
|
out = append(out, okFinding("inputs", item.name+": "+path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||||
|
if strings.TrimSpace(input.ConfigPath) == "" {
|
||||||
|
return "", fmt.Errorf("source config path is required")
|
||||||
|
}
|
||||||
|
path := strings.TrimSpace(input.Path)
|
||||||
|
if path == "" {
|
||||||
|
return "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(path) {
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLocalAudioFindings(cfg *config.Config) []finding {
|
||||||
|
if cfg.Session.Inputs.AudioS3 != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
audioDir := strings.TrimSpace(cfg.Session.Inputs.AudioDir)
|
||||||
|
if audioDir == "" && len(cfg.Session.Inputs.AudioFiles) == 0 {
|
||||||
|
return []finding{errorFinding("audio", "audio_dir, audio_files, or audio_s3 is required")}
|
||||||
|
}
|
||||||
|
base := filepath.Dir(cfg.SessionPath)
|
||||||
|
paths := []string{}
|
||||||
|
if audioDir != "" {
|
||||||
|
dir := audioDir
|
||||||
|
if !filepath.IsAbs(dir) {
|
||||||
|
dir = filepath.Join(base, dir)
|
||||||
|
}
|
||||||
|
matches, err := filepath.Glob(filepath.Join(dir, "*.flac"))
|
||||||
|
if err != nil || len(matches) == 0 {
|
||||||
|
return []finding{errorFinding("audio", "no .flac files found in "+dir)}
|
||||||
|
}
|
||||||
|
paths = append(paths, matches...)
|
||||||
|
}
|
||||||
|
for _, file := range cfg.Session.Inputs.AudioFiles {
|
||||||
|
p := file
|
||||||
|
if !filepath.IsAbs(p) {
|
||||||
|
p = filepath.Join(base, p)
|
||||||
|
}
|
||||||
|
paths = append(paths, p)
|
||||||
|
}
|
||||||
|
for _, p := range paths {
|
||||||
|
if _, err := os.Stat(p); err != nil {
|
||||||
|
return []finding{errorFinding("audio", fmt.Sprintf("audio file missing: %v", err))}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(paths)))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||||
|
objects, err := store.List(ctx, audioPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return errorFinding("audio", err.Error())
|
||||||
|
}
|
||||||
|
count := 0
|
||||||
|
for _, obj := range objects {
|
||||||
|
if strings.HasSuffix(strings.ToLower(obj.Key), ".flac") {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if count == 0 {
|
||||||
|
return errorFinding("audio", "no remote .flac objects found under "+audioPrefix)
|
||||||
|
}
|
||||||
|
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", count))
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePreviousArtifactFindings(ctx context.Context, cfg *config.Config, store storage.ObjectStore, requirements []artifacts.PreviousArtifactRequirement) []finding {
|
||||||
|
out := []finding{}
|
||||||
|
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
||||||
|
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(prefix)
|
||||||
|
for _, key := range []string{runIDKey, manifestKey} {
|
||||||
|
exists, err := store.Exists(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, errorFinding("previous", fmt.Sprintf("check %s: %v", key, err)))
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
out = append(out, errorFinding("previous", "missing "+key))
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, req := range requirements {
|
||||||
|
out = append(out, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
return store.Load(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
|
||||||
|
if m == nil || len(m.Stages) == 0 {
|
||||||
|
fmt.Fprintln(out, "stages: no stages recorded")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "stages:")
|
||||||
|
names := make([]string, 0, len(m.Stages))
|
||||||
|
for name := range m.Stages {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||||
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
||||||
|
if cfg.Pipeline.Scriptorium != nil {
|
||||||
|
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
||||||
|
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return catalog, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, promotedRemoteState map[string]string) {
|
||||||
|
lockSet := lockSourceSet(locks.All)
|
||||||
|
fmt.Fprintln(out, "Built-in:")
|
||||||
|
for _, id := range []string{
|
||||||
|
artifacts.ArtifactTranscriptMerged,
|
||||||
|
artifacts.ArtifactTranscriptPolished,
|
||||||
|
artifacts.ArtifactTranscriptFull,
|
||||||
|
artifacts.ArtifactTranscriptTrimmed,
|
||||||
|
artifacts.ArtifactBoundsSession,
|
||||||
|
} {
|
||||||
|
writeArtifactLine(out, id, lockSet)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Configured:")
|
||||||
|
for _, entry := range catalog.ListConfigured() {
|
||||||
|
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Previous-session:")
|
||||||
|
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||||
|
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Promoted:")
|
||||||
|
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||||
|
writePromotedArtifactLine(out, rule, catalog, lockSet, promotedRemoteState)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.ArchiveLockRule) {
|
||||||
|
parts := []string{source}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.ArchiveLockRule, remoteState map[string]string) {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
parts := []string{source}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
dest, showDest, err := helperPromotionDest(rule, catalog)
|
||||||
|
if err != nil {
|
||||||
|
parts = append(parts, "remote=error")
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if showDest {
|
||||||
|
parts = append(parts, "dest="+dest)
|
||||||
|
}
|
||||||
|
if state := remoteState[promotionRemoteStateKey(source, dest)]; state != "" {
|
||||||
|
parts = append(parts, state)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
|
func remotePromotionAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
dest, _, err := helperPromotionDest(rule, catalog)
|
||||||
|
if err != nil {
|
||||||
|
out[promotionRemoteStateKey(source, "")] = "remote=error"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := artifacts.S3PromotedArtifactKey(sessionPrefix, dest)
|
||||||
|
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||||
|
out[promotionRemoteStateKey(source, dest)] = "remote=promoted"
|
||||||
|
} else if err != nil {
|
||||||
|
out[promotionRemoteStateKey(source, dest)] = "remote=error"
|
||||||
|
} else {
|
||||||
|
out[promotionRemoteStateKey(source, dest)] = "remote=missing"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func helperPromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
dest := strings.TrimSpace(rule.Dest)
|
||||||
|
if dest == "" {
|
||||||
|
entry, ok := catalog.Lookup(source)
|
||||||
|
if !ok {
|
||||||
|
return "", false, fmt.Errorf("destination omitted and source is unknown")
|
||||||
|
}
|
||||||
|
dest = strings.TrimSpace(entry.CanonicalRelPath)
|
||||||
|
if dest == "" {
|
||||||
|
return "", false, fmt.Errorf("destination omitted and no canonical destination is available")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
normalized, err := normalizeHelperArchiveRelativePath(dest)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
entry, ok := catalog.Lookup(source)
|
||||||
|
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
|
||||||
|
return normalized, showDest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeHelperArchiveRelativePath(rel string) (string, error) {
|
||||||
|
trimmed := strings.TrimSpace(rel)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", fmt.Errorf("relative path is required")
|
||||||
|
}
|
||||||
|
cleaned := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
|
||||||
|
if cleaned == "." || cleaned == "" {
|
||||||
|
return "", fmt.Errorf("relative path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(trimmed) || strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
|
||||||
|
return "", fmt.Errorf("path must be a clean relative path")
|
||||||
|
}
|
||||||
|
return cleaned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promotionRemoteStateKey(source, dest string) string {
|
||||||
|
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||||
|
if locks == nil || len(locks.All) == 0 {
|
||||||
|
fmt.Fprintln(out, "Archive locks: none")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Archive locks:")
|
||||||
|
promoted := map[string]config.ArchivePromotionRule{}
|
||||||
|
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Archive != nil {
|
||||||
|
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||||
|
promoted[strings.TrimSpace(rule.Source)] = rule
|
||||||
|
}
|
||||||
|
}
|
||||||
|
staticSet := lockSourceSet(locks.Static)
|
||||||
|
for _, lock := range locks.All {
|
||||||
|
origin := "remote"
|
||||||
|
if _, ok := staticSet[lock.Source]; ok {
|
||||||
|
origin = "pipeline"
|
||||||
|
}
|
||||||
|
promo := "not-promoted"
|
||||||
|
if _, ok := promoted[lock.Source]; ok {
|
||||||
|
promo = "promoted"
|
||||||
|
}
|
||||||
|
reason := strings.TrimSpace(lock.Reason)
|
||||||
|
if reason == "" {
|
||||||
|
reason = "(no reason)"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockMapValues(in map[string]config.ArchiveLockRule) []config.ArchiveLockRule {
|
||||||
|
keys := make([]string, 0, len(in))
|
||||||
|
for key := range in {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := make([]config.ArchiveLockRule, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
item := in[key]
|
||||||
|
item.Source = key
|
||||||
|
item.Reason = strings.TrimSpace(item.Reason)
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
589
internal/app/operator_helpers_test.go
Normal file
589
internal/app/operator_helpers_test.go
Normal file
@@ -0,0 +1,589 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteSessionInitRemoteWritesCanonicalSessionConfig(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"session", "init",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session-id", "2026-06-07",
|
||||||
|
"--title", "The Black Cabin",
|
||||||
|
"--remote",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
key := artifacts.S3SessionConfigKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-06-07"))
|
||||||
|
obj, ok := fake.Objects[key]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("remote session key %q not uploaded; objects=%v", key, fake.Objects)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(obj.Data), `session_id: "2026-06-07"`) || !strings.Contains(string(obj.Data), "prefix: audio/") {
|
||||||
|
t.Fatalf("remote session data = %q", string(obj.Data))
|
||||||
|
}
|
||||||
|
if storeInitCalls != 1 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
|
||||||
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||||
|
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
audioKey := artifacts.S3PromotedArtifactKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if os.Getenv(accessKeyEnv) != "test-key-id" || os.Getenv(secretKeyEnv) != "test-secret" {
|
||||||
|
return nil, fmt.Errorf("secrets were not loaded before object store init")
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"session", "validate", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "OK audio") {
|
||||||
|
t.Fatalf("stdout = %q, want OK audio", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"locks", "add",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"--reason", "manual edit",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||||
|
obj, ok := fake.Objects[key]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("remote locks key %q not uploaded", key)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(obj.Data), "source: narratio.transcript.trimmed") || !strings.Contains(string(obj.Data), "reason: manual edit") {
|
||||||
|
t.Fatalf("lock store data = %q", string(obj.Data))
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"locks",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("locks list exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "- narratio.transcript.trimmed origin=remote") {
|
||||||
|
t.Fatalf("stdout = %q, want remote lock", stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"locks", "remove",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(store.Locks) != 0 {
|
||||||
|
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
||||||
|
}
|
||||||
|
if storeInitCalls != 3 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 3", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"locks", "add",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"--reason", "first",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("initial locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"locks", "add",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"--reason", "second",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("duplicate locks add exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "pass --force to update") {
|
||||||
|
t.Fatalf("stderr = %q, want force guidance", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"locks", "add",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"--reason", "second",
|
||||||
|
"--force",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("forced locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||||
|
if !strings.Contains(string(fake.Objects[key].Data), "reason: second") {
|
||||||
|
t.Fatalf("lock store data = %q, want updated reason", string(fake.Objects[key].Data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocksRequireSessionID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"list", []string{"locks"}, "locks: --session-id is required"},
|
||||||
|
{"add", []string{"locks", "add", "narratio.transcript.trimmed"}, "locks add: --session-id is required"},
|
||||||
|
{"remove", []string{"locks", "remove", "narratio.transcript.trimmed"}, "locks remove: --session-id is required"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(tt.args, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), tt.want) {
|
||||||
|
t.Fatalf("stderr = %q, want %q", stderr.String(), tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocksCannotModifyStaticLocks(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addStaticArchiveLockToPipelineConfig(t, pipelinePath, "narratio.transcript.trimmed")
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"locks", "add",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("locks add static lock exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "locked by pipeline config") {
|
||||||
|
t.Fatalf("stderr = %q, want static lock error", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"locks", "remove",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"narratio.transcript.trimmed",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("locks remove static lock exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "locked by pipeline config") {
|
||||||
|
t.Fatalf("stderr = %q, want static lock error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteTopLevelLockAndUnlockAreRemoved(t *testing.T) {
|
||||||
|
tests := []string{"lock", "unlock"}
|
||||||
|
for _, cmd := range tests {
|
||||||
|
t.Run(cmd, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{cmd, "narratio.transcript.trimmed"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `unknown command: "`+cmd+`"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown command", stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
|
promote_artifacts:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
trimmedKey := artifacts.S3PromotedArtifactKey(
|
||||||
|
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||||
|
"transcripts/trimmed.json",
|
||||||
|
)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"artifacts", "list",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--remote",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio.transcript.trimmed remote=promoted") {
|
||||||
|
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
|
promote_artifacts:
|
||||||
|
- source: narratio.transcript.full
|
||||||
|
dest: transcripts/full.json
|
||||||
|
required: true
|
||||||
|
- source: narratio.bounds.session
|
||||||
|
dest: transcripts/bounds.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"artifacts", "list",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--remote",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
out := stdout.String()
|
||||||
|
for _, unwanted := range []string{
|
||||||
|
"narratio.transcript.full remote=missing",
|
||||||
|
"narratio.bounds.session remote=missing",
|
||||||
|
} {
|
||||||
|
if strings.Contains(out, unwanted) {
|
||||||
|
t.Fatalf("stdout = %q, did not want catalog remote marker %q", out, unwanted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
|
||||||
|
"narratio.bounds.session dest=transcripts/bounds.json remote=promoted",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
|
promote_artifacts:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.full
|
||||||
|
dest: transcripts/full.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||||
|
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
||||||
|
trimmedKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
|
||||||
|
fullKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json")
|
||||||
|
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: fullKey, Data: []byte(`{"segments":[]}`)})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"status",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
out := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Remote outputs:",
|
||||||
|
"Built-in:",
|
||||||
|
"Configured:",
|
||||||
|
"Previous-session:",
|
||||||
|
"Promoted:",
|
||||||
|
"narratio.transcript.trimmed locked",
|
||||||
|
"narratio.transcript.trimmed locked remote=promoted",
|
||||||
|
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "narratio.transcript.merged remote=missing") {
|
||||||
|
t.Fatalf("stdout = %q, did not want catalog remote marker", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
|
promote_artifacts:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
fake := &storage.FakeBackend{ExistsErr: fmt.Errorf("exists failed")}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"status",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
out := stdout.String()
|
||||||
|
if !strings.Contains(out, "Remote archive: missing or unavailable:") {
|
||||||
|
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.trimmed remote=error") {
|
||||||
|
t.Fatalf("stdout = %q, want remote output error state", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "Archive locks: error:") {
|
||||||
|
t.Fatalf("stdout = %q, want archive locks error", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteArchiveLoadsRemoteLocks(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidArchiveConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
lockKey := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||||
|
// The archive stage only checks the manifest statuses and source files.
|
||||||
|
_ = stageName
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "archive"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
promotedKey := artifacts.S3PromotedArtifactKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/trimmed.json")
|
||||||
|
if _, ok := fake.Objects[promotedKey]; ok {
|
||||||
|
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addArchivePromotionsToPipeline(t *testing.T, pipelinePath, archiveYAML string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read pipeline: %v", err)
|
||||||
|
}
|
||||||
|
updated := strings.Replace(string(data), " upload_run: false\n", " upload_run: false\n"+archiveYAML, 1)
|
||||||
|
if updated == string(data) {
|
||||||
|
t.Fatalf("pipeline %q did not contain archive upload_run marker", pipelinePath)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidArchiveConfigFiles(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read pipeline: %v", err)
|
||||||
|
}
|
||||||
|
updated := strings.Replace(string(data), "upload_run: false", "upload_run: true", 1)
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
m := manifest.New("2026-05-03", nowUTC())
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260521T160000Z-test"
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||||
|
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||||
|
}
|
||||||
|
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if err := store.Save(ctx, path, m); err != nil {
|
||||||
|
t.Fatalf("save manifest: %v", err)
|
||||||
|
}
|
||||||
|
runManifestPath := artifacts.SessionRunManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, m.RunID)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(runManifestPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir run manifest: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(runManifestPath, []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write run manifest: %v", err)
|
||||||
|
}
|
||||||
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func addStaticArchiveLockToPipelineConfig(t *testing.T, pipelinePath, source string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read pipeline: %v", err)
|
||||||
|
}
|
||||||
|
updated := strings.Replace(
|
||||||
|
string(data),
|
||||||
|
"archive:\n enabled: true\n upload_run: false\n",
|
||||||
|
"archive:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
if updated == string(data) {
|
||||||
|
t.Fatalf("archive section not found in pipeline config")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -20,11 +20,13 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var previousSessionID string
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
@@ -36,16 +38,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("plan: unexpected positional arguments")
|
return fmt.Errorf("plan: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("plan: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("plan: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
PreviousSessionID: previousSessionID,
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -15,10 +15,10 @@ import (
|
|||||||
|
|
||||||
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
args := []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}
|
||||||
|
|
||||||
if err := Plan(context.Background(), args, &out); err != nil {
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
t.Fatalf("first Plan() error = %v", err)
|
t.Fatalf("first Plan() error = %v", err)
|
||||||
@@ -62,7 +62,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
|
|
||||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
|
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out); err != nil {
|
||||||
t.Fatalf("Plan() error = %v", err)
|
t.Fatalf("Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
got := out.String()
|
got := out.String()
|
||||||
@@ -93,6 +93,7 @@ func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -128,7 +129,7 @@ inputs:
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -153,53 +153,70 @@ func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool,
|
|||||||
return true, ""
|
return true, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type scopedDir struct {
|
||||||
|
RootAbs string
|
||||||
|
TargetAbs string
|
||||||
|
Exists bool
|
||||||
|
}
|
||||||
|
|
||||||
func removeRunScopedDir(root, target, policy string) error {
|
func removeRunScopedDir(root, target, policy string) error {
|
||||||
|
dir, err := validateScopedDir(root, target, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !dir.Exists {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||||
cleanRoot := strings.TrimSpace(root)
|
cleanRoot := strings.TrimSpace(root)
|
||||||
cleanTarget := strings.TrimSpace(target)
|
cleanTarget := strings.TrimSpace(target)
|
||||||
if cleanRoot == "" {
|
if cleanRoot == "" {
|
||||||
return fmt.Errorf("cleanup policy %s: root path is required", policy)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
}
|
}
|
||||||
if cleanTarget == "" {
|
if cleanTarget == "" {
|
||||||
return fmt.Errorf("cleanup policy %s: target path is required", policy)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||||
}
|
}
|
||||||
|
|
||||||
rootAbs, err := filepath.Abs(cleanRoot)
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
}
|
}
|
||||||
targetAbs, err := filepath.Abs(cleanTarget)
|
targetAbs, err := filepath.Abs(cleanTarget)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rel, err := filepath.Rel(rootAbs, targetAbs)
|
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||||
}
|
}
|
||||||
if rel == "." {
|
if rel == "." {
|
||||||
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||||
}
|
}
|
||||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||||
}
|
}
|
||||||
|
|
||||||
info, err := os.Lstat(targetAbs)
|
info, err := os.Lstat(targetAbs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||||
}
|
}
|
||||||
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||||
}
|
}
|
||||||
if info.Mode()&os.ModeSymlink != 0 {
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||||
}
|
}
|
||||||
if !info.IsDir() {
|
if !info.IsDir() {
|
||||||
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(targetAbs); err != nil {
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func asString(v any) string {
|
func asString(v any) string {
|
||||||
|
|||||||
157
internal/app/remote_locks.go
Normal file
157
internal/app/remote_locks.go
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type effectiveLocks struct {
|
||||||
|
Static []config.ArchiveLockRule
|
||||||
|
Remote []config.ArchiveLockRule
|
||||||
|
All []config.ArchiveLockRule
|
||||||
|
Key string
|
||||||
|
}
|
||||||
|
|
||||||
|
func remoteLocksKey(cfg *config.Config) (string, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
|
return "", fmt.Errorf("resolved config is required")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil {
|
||||||
|
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(
|
||||||
|
cfg.Pipeline.Storage.S3.RootPrefix,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
)
|
||||||
|
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.ArchiveLockStore, string, error) {
|
||||||
|
key, err := remoteLocksKey(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
exists, err := store.Exists(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return &config.ArchiveLockStore{}, key, nil
|
||||||
|
}
|
||||||
|
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(tmp) }()
|
||||||
|
data, err := os.ReadFile(tmp)
|
||||||
|
if err != nil {
|
||||||
|
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||||
|
}
|
||||||
|
lockStore, err := config.LoadArchiveLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
||||||
|
if err != nil {
|
||||||
|
return nil, key, err
|
||||||
|
}
|
||||||
|
return lockStore, key, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
||||||
|
staticLocks := staticArchiveLocks(cfg)
|
||||||
|
if store == nil {
|
||||||
|
return &effectiveLocks{
|
||||||
|
Static: staticLocks,
|
||||||
|
All: append([]config.ArchiveLockRule(nil), staticLocks...),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
remoteLocks := append([]config.ArchiveLockRule(nil), lockStore.Locks...)
|
||||||
|
return &effectiveLocks{
|
||||||
|
Static: staticLocks,
|
||||||
|
Remote: remoteLocks,
|
||||||
|
All: config.MergeArchiveLockRules(staticLocks, remoteLocks),
|
||||||
|
Key: key,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func staticArchiveLocks(cfg *config.Config) []config.ArchiveLockRule {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return append([]config.ArchiveLockRule(nil), cfg.Pipeline.Archive.Locks...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyEffectiveLocks(cfg *config.Config, locks []config.ArchiveLockRule) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive == nil {
|
||||||
|
cfg.Pipeline.Archive = &config.ArchiveConfig{}
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Archive.Locks = append([]config.ArchiveLockRule(nil), locks...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.ArchiveLockStore) error {
|
||||||
|
data, err := config.MarshalArchiveLockStore(lockStore)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create lock store temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
defer func() { _ = os.Remove(tmpPath) }()
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("write lock store temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close lock store temp file: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||||
|
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockSourceSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||||
|
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||||
|
for _, lock := range locks {
|
||||||
|
source := strings.TrimSpace(lock.Source)
|
||||||
|
if source == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lock.Source = source
|
||||||
|
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||||
|
out[source] = lock
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLocalFile(path string, data []byte, force bool) error {
|
||||||
|
cleaned := filepath.Clean(strings.TrimSpace(path))
|
||||||
|
if cleaned == "" || cleaned == "." {
|
||||||
|
return fmt.Errorf("output path is required")
|
||||||
|
}
|
||||||
|
if !force {
|
||||||
|
if _, err := os.Stat(cleaned); err == nil {
|
||||||
|
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
|
||||||
|
} else if err != nil && !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create output directory: %w", err)
|
||||||
|
}
|
||||||
|
return os.WriteFile(cleaned, data, 0o644)
|
||||||
|
}
|
||||||
252
internal/app/remote_session_test.go
Normal file
252
internal/app/remote_session_test.go
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 1 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio plan: workdir prepared") {
|
||||||
|
t.Fatalf("stdout = %q, want plan output", stdout.String())
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[remoteKey]; !ok {
|
||||||
|
t.Fatalf("remote session key %q was not seeded", remoteKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
|
||||||
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
|
||||||
|
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
|
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||||
|
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||||
|
})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
|
||||||
|
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), missingSessionPath) {
|
||||||
|
t.Fatalf("stderr = %q, want local searched path", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "remote session loading requires --session-id") {
|
||||||
|
t.Fatalf("stderr = %q, want session-id guidance", stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
|
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
return nil, errors.New("storage unavailable")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||||
|
})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
|
||||||
|
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "strict decode failed") {
|
||||||
|
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
|
||||||
|
t.Helper()
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
|
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if storeInitCalls != nil {
|
||||||
|
(*storeInitCalls)++
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
|
||||||
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
fake.SeedObject(storage.FakeObject{
|
||||||
|
Key: remoteKey,
|
||||||
|
Data: []byte(content),
|
||||||
|
ETag: "remote-session-etag",
|
||||||
|
})
|
||||||
|
return remoteKey
|
||||||
|
}
|
||||||
|
|
||||||
|
func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv string) {
|
||||||
|
t.Helper()
|
||||||
|
pipelineData, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read pipeline: %v", err)
|
||||||
|
}
|
||||||
|
pipelineYAML := strings.Replace(
|
||||||
|
string(pipelineData),
|
||||||
|
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
||||||
|
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(out)
|
fs.SetOutput(out)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var previousSessionID string
|
var previousSessionID string
|
||||||
@@ -33,6 +34,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
var force bool
|
var force bool
|
||||||
var includeAudio bool
|
var includeAudio bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
@@ -40,7 +42,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
||||||
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
||||||
fs.Usage = func() {
|
fs.Usage = func() {
|
||||||
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--campaign <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
||||||
_, _ = fmt.Fprintln(out)
|
_, _ = fmt.Fprintln(out)
|
||||||
_, _ = fmt.Fprintln(out, "Flags:")
|
_, _ = fmt.Fprintln(out, "Flags:")
|
||||||
fs.PrintDefaults()
|
fs.PrintDefaults()
|
||||||
@@ -55,16 +57,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("restore: unexpected positional arguments")
|
return fmt.Errorf("restore: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
PreviousSessionID: previousSessionID,
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
@@ -74,11 +67,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
|
objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
objectStore, err := newObjectStoreFromConfigFn(ctx, cfg)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/audio"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
@@ -92,6 +93,10 @@ func executeRestoreDownloadAction(
|
|||||||
return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath)
|
return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if restoreActionIsAudio(action) {
|
||||||
|
return executeRestoreAudioAction(ctx, cfg, safeLocalPath, action, store)
|
||||||
|
}
|
||||||
|
|
||||||
tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath)
|
tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("download to temp file: %w", err)
|
return fmt.Errorf("download to temp file: %w", err)
|
||||||
@@ -120,6 +125,38 @@ func executeRestoreDownloadAction(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func executeRestoreAudioAction(
|
||||||
|
ctx context.Context,
|
||||||
|
cfg *config.Config,
|
||||||
|
safeLocalPath string,
|
||||||
|
action RestoreAction,
|
||||||
|
store storage.ObjectStore,
|
||||||
|
) error {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || cfg.Session == nil {
|
||||||
|
return fmt.Errorf("resolved s3 config and session are required")
|
||||||
|
}
|
||||||
|
spoolDir := artifacts.SessionSpoolRestoreAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
spoolPath := filepath.Join(spoolDir, filepath.Base(safeLocalPath))
|
||||||
|
cacheEnabled := cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio
|
||||||
|
_, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||||
|
Store: store,
|
||||||
|
Object: storage.ObjectInfo{
|
||||||
|
Key: action.RemoteKey,
|
||||||
|
Size: action.Size,
|
||||||
|
ETag: action.ETag,
|
||||||
|
},
|
||||||
|
Bucket: strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket),
|
||||||
|
CacheRoot: strings.TrimSpace(cfg.Pipeline.Cache.Root),
|
||||||
|
CacheEnabled: cacheEnabled,
|
||||||
|
SpoolPath: spoolPath,
|
||||||
|
DestPath: safeLocalPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("materialize audio: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) {
|
func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) {
|
||||||
if strings.TrimSpace(destPath) == "" {
|
if strings.TrimSpace(destPath) == "" {
|
||||||
return "", fmt.Errorf("destination path is required")
|
return "", fmt.Errorf("destination path is required")
|
||||||
|
|||||||
@@ -18,10 +18,10 @@ import (
|
|||||||
|
|
||||||
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
|
||||||
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
|
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
|
||||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||||
@@ -32,7 +32,7 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -64,17 +64,17 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||||
|
|
||||||
restoreWithStoreAndRealPhases(t, fake)
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -87,12 +87,50 @@ func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
func TestExecuteRestoreIncludeAudioUsesCacheAfterWorkspaceDeletion(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
|
audioKey := sessionPrefix + "audio/alice.flac"
|
||||||
|
seedRestoreObject(fake, audioKey, []byte("remote-audio"))
|
||||||
|
|
||||||
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("first restore exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if got := fakeDownloadCount(fake, audioKey); got != 1 {
|
||||||
|
t.Fatalf("audio downloads after first restore = %d, want 1", got)
|
||||||
|
}
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
|
||||||
|
|
||||||
|
if err := os.RemoveAll(sessionRoot); err != nil {
|
||||||
|
t.Fatalf("remove session root: %v", err)
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("second restore exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if got := fakeDownloadCount(fake, audioKey); got != 1 {
|
||||||
|
t.Fatalf("audio downloads after cached restore = %d, want still 1", got)
|
||||||
|
}
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
||||||
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
||||||
|
|
||||||
@@ -100,7 +138,7 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -116,10 +154,10 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -129,7 +167,7 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -148,10 +186,10 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -161,7 +199,7 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -174,10 +212,10 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
|
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
|
||||||
|
|
||||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -187,7 +225,7 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -196,10 +234,10 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspaceRoot)
|
store := artifacts.NewLocalStore(workspaceRoot)
|
||||||
@@ -213,7 +251,7 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -229,10 +267,10 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
|
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
base := &storage.FakeBackend{}
|
base := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
toggled := &stagedManifestDownloadStore{
|
toggled := &stagedManifestDownloadStore{
|
||||||
@@ -260,7 +298,7 @@ func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -344,9 +382,9 @@ func restoreWithStoreAndRealPhases(t *testing.T, objectStore storage.ObjectStore
|
|||||||
executeRestorePlanFn = executeRestorePlan
|
executeRestorePlanFn = executeRestorePlan
|
||||||
}
|
}
|
||||||
|
|
||||||
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, sessionPath string) (*config.Config, string, string, string) {
|
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, campaignPath, sessionPath string) (*config.Config, string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, sessionPath, config.SessionLoadOptions{})
|
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -374,6 +412,16 @@ func mustReadEquals(t *testing.T, path, want string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fakeDownloadCount(fake *storage.FakeBackend, key string) int {
|
||||||
|
count := 0
|
||||||
|
for _, call := range fake.Downloads {
|
||||||
|
if call.Key == key {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
type stagedManifestDownloadStore struct {
|
type stagedManifestDownloadStore struct {
|
||||||
delegate *storage.FakeBackend
|
delegate *storage.FakeBackend
|
||||||
manifestKey string
|
manifestKey string
|
||||||
|
|||||||
@@ -257,6 +257,35 @@ func classifyRestoreAction(
|
|||||||
return action, nil
|
return action, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if restoreRelativePathIsAudio(localRelPath) {
|
||||||
|
if object.Size > 0 {
|
||||||
|
if info.Size() == object.Size {
|
||||||
|
action.Kind = RestoreActionSkipSame
|
||||||
|
action.SameLocal = true
|
||||||
|
action.Reason = "local audio size matches remote content"
|
||||||
|
return action, nil
|
||||||
|
}
|
||||||
|
if force {
|
||||||
|
action.Kind = RestoreActionDownload
|
||||||
|
action.Reason = "local audio differs (size mismatch); overwrite with --force"
|
||||||
|
return action, nil
|
||||||
|
}
|
||||||
|
action.Kind = RestoreActionConflict
|
||||||
|
action.Conflict = true
|
||||||
|
action.Reason = "local audio differs (size mismatch)"
|
||||||
|
return action, nil
|
||||||
|
}
|
||||||
|
if force {
|
||||||
|
action.Kind = RestoreActionDownload
|
||||||
|
action.Reason = "local audio exists; remote size unavailable; overwrite with --force"
|
||||||
|
return action, nil
|
||||||
|
}
|
||||||
|
action.Kind = RestoreActionConflict
|
||||||
|
action.Conflict = true
|
||||||
|
action.Reason = "local audio exists; remote size unavailable"
|
||||||
|
return action, nil
|
||||||
|
}
|
||||||
|
|
||||||
if object.Size > 0 && info.Size() != object.Size {
|
if object.Size > 0 && info.Size() != object.Size {
|
||||||
if force {
|
if force {
|
||||||
action.Kind = RestoreActionDownload
|
action.Kind = RestoreActionDownload
|
||||||
@@ -303,6 +332,15 @@ func classifyRestoreAction(
|
|||||||
return action, nil
|
return action, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func restoreActionIsAudio(action RestoreAction) bool {
|
||||||
|
return restoreRelativePathIsAudio(action.LocalRelativePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreRelativePathIsAudio(rel string) bool {
|
||||||
|
cleanRel := path.Clean(strings.TrimSpace(rel))
|
||||||
|
return cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")
|
||||||
|
}
|
||||||
|
|
||||||
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
|
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
|
||||||
if out == nil {
|
if out == nil {
|
||||||
return fmt.Errorf("output writer is required")
|
return fmt.Errorf("output writer is required")
|
||||||
|
|||||||
@@ -58,6 +58,33 @@ func TestRestorePlanIncludeAudio(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestorePlanExistingAudioUsesSizeWithoutRemoteChecksumDownload(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
current := restorePlanCurrentState(t, cfg)
|
||||||
|
store := &storage.FakeBackend{}
|
||||||
|
|
||||||
|
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
|
||||||
|
seedRestoreObject(store, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustWriteTestFile(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "local")
|
||||||
|
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{IncludeAudio: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(store.Downloads) != 0 {
|
||||||
|
t.Fatalf("downloads = %d, want no remote checksum download for audio", len(store.Downloads))
|
||||||
|
}
|
||||||
|
actionByRel := map[string]RestoreAction{}
|
||||||
|
for _, action := range plan.Actions {
|
||||||
|
actionByRel[action.LocalRelativePath] = action
|
||||||
|
}
|
||||||
|
audioAction := actionByRel["audio/alice.flac"]
|
||||||
|
if audioAction.Kind != RestoreActionSkipSame {
|
||||||
|
t.Fatalf("audio action kind = %q, want %q", audioAction.Kind, RestoreActionSkipSame)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
|
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
|
||||||
cfg := restorePlanConfig(t)
|
cfg := restorePlanConfig(t)
|
||||||
current := restorePlanCurrentState(t, cfg)
|
current := restorePlanCurrentState(t, cfg)
|
||||||
|
|||||||
@@ -32,6 +32,9 @@ func TestExecuteRestoreHelp(t *testing.T) {
|
|||||||
if !strings.Contains(out, "--include-audio") {
|
if !strings.Contains(out, "--include-audio") {
|
||||||
t.Fatalf("stdout = %q, want --include-audio flag", out)
|
t.Fatalf("stdout = %q, want --include-audio flag", out)
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(out, "--campaign") {
|
||||||
|
t.Fatalf("stdout = %q, want --campaign flag", out)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||||
@@ -70,7 +73,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -78,6 +81,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", "2026-05-03",
|
"--session-id", "2026-05-03",
|
||||||
"--dry-run",
|
"--dry-run",
|
||||||
@@ -116,11 +120,11 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
|
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -138,11 +142,11 @@ func TestExecuteRestoreFailsWhenStorageBackendNotConfigured(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -166,11 +170,11 @@ func TestExecuteRestoreDiscoveryErrorSurfaced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -208,7 +212,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
restoreEnv(secretKeyEnv)
|
restoreEnv(secretKeyEnv)
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
||||||
@@ -261,6 +265,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", "2026-05-03",
|
"--session-id", "2026-05-03",
|
||||||
"--dry-run",
|
"--dry-run",
|
||||||
@@ -307,10 +312,10 @@ func TestExecuteRestoreNonDryRunConflictFailsBeforeNYI(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -359,10 +364,10 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -374,11 +379,12 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string) {
|
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, dir)
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -405,5 +411,5 @@ inputs:
|
|||||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ import (
|
|||||||
|
|
||||||
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||||
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
|
||||||
@@ -48,6 +48,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
},
|
},
|
||||||
@@ -86,6 +87,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"run-stage",
|
"run-stage",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
"--force",
|
"--force",
|
||||||
@@ -157,7 +159,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
|
|
||||||
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
|
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
|
||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
binary: scriptorium
|
||||||
@@ -176,7 +178,7 @@ scriptorium:
|
|||||||
`)
|
`)
|
||||||
|
|
||||||
fakeStore := &storage.FakeBackend{}
|
fakeStore := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||||
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||||
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
|
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
|
||||||
@@ -191,6 +193,7 @@ scriptorium:
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
},
|
},
|
||||||
@@ -235,6 +238,7 @@ scriptorium:
|
|||||||
[]string{
|
[]string{
|
||||||
"run-stage",
|
"run-stage",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
"--force",
|
"--force",
|
||||||
|
|||||||
@@ -17,12 +17,14 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var previousSessionID string
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
@@ -35,16 +37,7 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("resume: unexpected positional arguments")
|
return fmt.Errorf("resume: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("resume: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("resume: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
PreviousSessionID: previousSessionID,
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -32,7 +32,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -51,7 +51,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
|
|
||||||
func TestResumeNoRemainingStages(t *testing.T) {
|
func TestResumeNoRemainingStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -64,7 +64,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -80,7 +80,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
|
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -93,7 +93,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force"}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -104,14 +104,14 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -134,7 +134,7 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageSkipAndForce(t *testing.T) {
|
func TestRunStageSkipAndForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
@@ -148,7 +148,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -157,7 +157,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
out.Reset()
|
out.Reset()
|
||||||
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(force) error = %v", err)
|
t.Fatalf("RunStage(force) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -168,7 +168,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
@@ -184,7 +184,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(force) error = %v", err)
|
t.Fatalf("RunStage(force) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -203,7 +203,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
out.Reset()
|
out.Reset()
|
||||||
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err = Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -214,13 +214,13 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
|
|
||||||
func TestRunStageTrimExecutes(t *testing.T) {
|
func TestRunStageTrimExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "trim"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "trim"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(trim) error = %v", err)
|
t.Fatalf("RunStage(trim) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -243,13 +243,13 @@ func TestRunStageTrimExecutes(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageNormalizeExecutes(t *testing.T) {
|
func TestRunStageNormalizeExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "normalize"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(normalize) error = %v", err)
|
t.Fatalf("RunStage(normalize) error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,12 +15,14 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var previousSessionID string
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
@@ -33,16 +35,7 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("run: unexpected positional arguments")
|
return fmt.Errorf("run: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
PreviousSessionID: previousSessionID,
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -15,12 +15,14 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var previousSessionID string
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
@@ -33,54 +35,34 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 1 {
|
if fs.NArg() != 1 {
|
||||||
return fmt.Errorf("run-stage: expected exactly one stage name")
|
return fmt.Errorf("run-stage: expected exactly one stage name")
|
||||||
}
|
}
|
||||||
stageName := fs.Arg(0)
|
|
||||||
normalizedArtifacts, err := selectedArtifacts.Normalize()
|
normalizedArtifacts, err := selectedArtifacts.Normalize()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
|
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
|
||||||
}
|
}
|
||||||
|
stageName := fs.Arg(0)
|
||||||
if len(normalizedArtifacts) > 0 && stageName != "analyze" {
|
if len(normalizedArtifacts) > 0 && stageName != "analyze" {
|
||||||
return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"")
|
return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"")
|
||||||
}
|
}
|
||||||
stages, err := BuildSingleStagePlan(stageName)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||||
if err != nil {
|
CommandName: "run-stage",
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
StageName: stageName,
|
||||||
}
|
PipelinePath: pipelinePath,
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
CampaignPath: campaignPath,
|
||||||
if err != nil {
|
SessionPath: sessionPath,
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
PreviousSessionID: previousSessionID,
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
if err := config.Validate(cfg); err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
if err := validateSelectedAnalyzeArtifacts(cfg, normalizedArtifacts); err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
|
|
||||||
Force: force,
|
Force: force,
|
||||||
SelectedArtifacts: normalizedArtifacts,
|
SelectedArtifacts: normalizedArtifacts,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = fmt.Fprintf(
|
_, err = fmt.Fprintf(
|
||||||
out,
|
out,
|
||||||
"narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n",
|
"narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n",
|
||||||
stages[0].Name(),
|
stageName,
|
||||||
len(summary.Executed),
|
len(summary.Executed),
|
||||||
len(summary.Skipped),
|
len(summary.Skipped),
|
||||||
force,
|
force,
|
||||||
@@ -88,3 +70,99 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Analyze force-runs the analyze stage.
|
||||||
|
func Analyze(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("analyze", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
|
var sessionPath string
|
||||||
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
|
var selectedArtifacts artifactSelectionFlag
|
||||||
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
|
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
||||||
|
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return fmt.Errorf("analyze: invalid flags: %w", err)
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return fmt.Errorf("analyze: unexpected positional arguments")
|
||||||
|
}
|
||||||
|
normalizedArtifacts, err := selectedArtifacts.Normalize()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("analyze: invalid --artifacts: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||||
|
CommandName: "analyze",
|
||||||
|
StageName: "analyze",
|
||||||
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
|
SessionPath: sessionPath,
|
||||||
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
|
Force: true,
|
||||||
|
SelectedArtifacts: normalizedArtifacts,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = fmt.Fprintf(
|
||||||
|
out,
|
||||||
|
"narratio analyze: executed=%d skipped=%d force=true; manifest=%s\n",
|
||||||
|
len(summary.Executed),
|
||||||
|
len(summary.Skipped),
|
||||||
|
summary.ManifestPath,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type singleStageCommand struct {
|
||||||
|
CommandName string
|
||||||
|
StageName string
|
||||||
|
PipelinePath string
|
||||||
|
CampaignPath string
|
||||||
|
SessionPath string
|
||||||
|
SessionID string
|
||||||
|
PreviousSessionID string
|
||||||
|
Force bool
|
||||||
|
SelectedArtifacts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func runSingleStageCommand(ctx context.Context, req singleStageCommand) (*RunSummary, error) {
|
||||||
|
stages, err := BuildSingleStagePlan(req.StageName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := loadCommandConfig(ctx, req.PipelinePath, req.CampaignPath, req.SessionPath, config.SessionLoadOptions{
|
||||||
|
SessionID: req.SessionID,
|
||||||
|
PreviousSessionID: req.PreviousSessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
|
}
|
||||||
|
if err := validateSelectedAnalyzeArtifacts(cfg, req.SelectedArtifacts); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
|
||||||
|
Force: req.Force,
|
||||||
|
SelectedArtifacts: req.SelectedArtifacts,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
|
}
|
||||||
|
return summary, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -87,12 +87,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
env.Storage = &storage.NoopBackend{}
|
env.Storage = &storage.NoopBackend{}
|
||||||
}
|
}
|
||||||
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
|
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
|
||||||
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
|
objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
return nil, err
|
||||||
}
|
}
|
||||||
env.ObjectStore = objectStore
|
env.ObjectStore = objectStore
|
||||||
}
|
}
|
||||||
|
if needsRemoteLocksForRun(env.Config, stages) {
|
||||||
|
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load remote archive locks: %w", err)
|
||||||
|
}
|
||||||
|
applyEffectiveLocks(env.Config, locks.All)
|
||||||
|
}
|
||||||
if env.Notifier == nil {
|
if env.Notifier == nil {
|
||||||
env.Notifier = ¬ify.NoopSender{}
|
env.Notifier = ¬ify.NoopSender{}
|
||||||
}
|
}
|
||||||
@@ -576,6 +583,32 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
archiveRequested := false
|
||||||
|
for _, s := range stages {
|
||||||
|
if s != nil && s.Name() == "archive" {
|
||||||
|
archiveRequested = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !archiveRequested {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return cfg.Pipeline.Storage.S3 != nil
|
||||||
|
}
|
||||||
|
|
||||||
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
|
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -994,10 +994,12 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
cfgDir := t.TempDir()
|
cfgDir := t.TempDir()
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
|
||||||
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
mustWriteFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
|
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
@@ -1005,8 +1007,27 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
|
|
||||||
return &config.Config{
|
return &config.Config{
|
||||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
@@ -1023,6 +1044,7 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + t.TempDir() + `
|
root: ` + t.TempDir() + `
|
||||||
@@ -1032,6 +1054,12 @@ analyzer:
|
|||||||
timeout: 20m
|
timeout: 20m
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
|
`
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
`
|
`
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
@@ -1042,6 +1070,7 @@ inputs:
|
|||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`
|
`
|
||||||
mustWriteFile(t, pipelinePath, pipelineYAML)
|
mustWriteFile(t, pipelinePath, pipelineYAML)
|
||||||
|
mustWriteFile(t, campaignPath, campaignYAML)
|
||||||
mustWriteFile(t, sessionPath, sessionYAML)
|
mustWriteFile(t, sessionPath, sessionYAML)
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
cfg, err := config.Load(pipelinePath, sessionPath)
|
||||||
|
|||||||
@@ -4,14 +4,15 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
|
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
sessionTemplate := `session_id: "{{ session_id }}"
|
sessionTemplate := `session_id: "{{ session_id }}"
|
||||||
previous_session_id: "{{ previous_session_id }}"
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
@@ -26,19 +27,14 @@ inputs:
|
|||||||
t.Fatalf("write session template: %v", err)
|
t.Fatalf("write session template: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cwd := filepath.Dir(sessionPath)
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
originalWD, err := os.Getwd()
|
config.DefaultSessionConfigSearchPaths = []string{sessionPath}
|
||||||
if err != nil {
|
t.Cleanup(func() { config.DefaultSessionConfigSearchPaths = origSessionDefaults })
|
||||||
t.Fatalf("Getwd(): %v", err)
|
|
||||||
}
|
|
||||||
if err := os.Chdir(cwd); err != nil {
|
|
||||||
t.Fatalf("Chdir(%q): %v", cwd, err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := Plan(context.Background(), []string{
|
if err := Plan(context.Background(), []string{
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session-id", "2026-04-04",
|
"--session-id", "2026-04-04",
|
||||||
"--previous-session-id", "2026-03-28",
|
"--previous-session-id", "2026-03-28",
|
||||||
}, &out); err != nil {
|
}, &out); err != nil {
|
||||||
@@ -51,10 +47,10 @@ inputs:
|
|||||||
|
|
||||||
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
@@ -65,7 +61,7 @@ func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
|||||||
|
|
||||||
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
|
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
previous_session_id: 2026-04-26
|
previous_session_id: 2026-04-26
|
||||||
@@ -83,6 +79,7 @@ inputs:
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{
|
err := Plan(context.Background(), []string{
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", "2026-05-03",
|
"--session-id", "2026-05-03",
|
||||||
"--previous-session-id", "2026-04-25",
|
"--previous-session-id", "2026-04-25",
|
||||||
@@ -97,10 +94,10 @@ inputs:
|
|||||||
|
|
||||||
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -110,7 +107,7 @@ func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||||
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
|
_, err := resolveSessionConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,22 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
|||||||
return explicit, nil
|
return explicit, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolved, err := discoverSessionConfigPathWithCandidates(candidates)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if resolved.Path != "" {
|
||||||
|
return resolved.Path, nil
|
||||||
|
}
|
||||||
|
return "", missingSessionConfigError(resolved.Searched, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
type sessionConfigDiscovery struct {
|
||||||
|
Path string
|
||||||
|
Searched []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func discoverSessionConfigPathWithCandidates(candidates []string) (sessionConfigDiscovery, error) {
|
||||||
ordered := make([]string, 0, len(candidates))
|
ordered := make([]string, 0, len(candidates))
|
||||||
for _, raw := range candidates {
|
for _, raw := range candidates {
|
||||||
path := strings.TrimSpace(raw)
|
path := strings.TrimSpace(raw)
|
||||||
@@ -31,19 +47,32 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
|||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return filepath.Clean(path), nil
|
return sessionConfigDiscovery{Path: filepath.Clean(path), Searched: ordered}, nil
|
||||||
}
|
}
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("check default session config %q: %w", path, err)
|
return sessionConfigDiscovery{}, fmt.Errorf("check default session config %q: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return sessionConfigDiscovery{Searched: ordered}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func missingSessionConfigError(searched []string, remoteDetail string) error {
|
||||||
|
ordered := append([]string(nil), searched...)
|
||||||
if len(ordered) == 0 {
|
if len(ordered) == 0 {
|
||||||
return "", fmt.Errorf("no session config path provided and no default locations configured")
|
if strings.TrimSpace(remoteDetail) != "" {
|
||||||
|
return fmt.Errorf("no session config path provided and no default locations configured; %s", remoteDetail)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("no session config path provided and no default locations configured")
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf(
|
msg := fmt.Sprintf(
|
||||||
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
|
"no session config path provided and no default session config found; searched: %s",
|
||||||
strings.Join(ordered, ", "),
|
strings.Join(ordered, ", "),
|
||||||
)
|
)
|
||||||
|
if strings.TrimSpace(remoteDetail) != "" {
|
||||||
|
msg += "; " + strings.TrimSpace(remoteDetail)
|
||||||
|
}
|
||||||
|
msg += "; pass --session to use an explicit path"
|
||||||
|
return fmt.Errorf("%s", msg)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Status reads and prints stage statuses from an existing manifest.
|
|
||||||
func Status(ctx context.Context, args []string, out io.Writer) error {
|
|
||||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
|
||||||
fs.SetOutput(io.Discard)
|
|
||||||
|
|
||||||
var manifestPath string
|
|
||||||
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
|
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
|
||||||
return fmt.Errorf("status: invalid flags: %w", err)
|
|
||||||
}
|
|
||||||
if fs.NArg() != 0 {
|
|
||||||
return fmt.Errorf("status: unexpected positional arguments")
|
|
||||||
}
|
|
||||||
if manifestPath == "" {
|
|
||||||
return fmt.Errorf("status: --manifest is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
|
||||||
m, err := store.Load(ctx, manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("status: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(m.Stages) == 0 {
|
|
||||||
_, err := fmt.Fprintln(out, "stages: no stages recorded")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := fmt.Fprintln(out, "stages:"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make([]string, 0, len(m.Stages))
|
|
||||||
for name := range m.Stages {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
status := m.Stages[name].Status
|
|
||||||
if _, err := fmt.Fprintf(out, "- %s: %s\n", name, status); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -409,6 +410,17 @@ func TestResolvePreviousSessionArtifactWithCatalogFallsBackToPreparedCachePath(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreviousSessionCacheCandidatePathsStripsArtifactsPrefix(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|
||||||
|
got := previousSessionCacheCandidatePaths(paths, "artifacts/session_recap.md")
|
||||||
|
want := []string{filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("previousSessionCacheCandidatePaths() = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
|
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
@@ -64,7 +67,7 @@ func SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID string)
|
|||||||
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
|
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
|
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
|
||||||
filepath.FromSlash(artifactRelativePath),
|
filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +91,60 @@ func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
|||||||
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SessionSpoolDir returns the campaign/session scoped local spool root.
|
||||||
|
func SessionSpoolDir(spoolRoot, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(spoolRoot, campaign, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionSpoolRestoreAudioDir returns the local spool audio path for restore downloads.
|
||||||
|
func SessionSpoolRestoreAudioDir(spoolRoot, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(SessionSpoolDir(spoolRoot, campaign, sessionID), "restore", config.PathAudioDirSegment)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3AudioCachePath returns the durable local cache path for one S3 audio object.
|
||||||
|
func S3AudioCachePath(cacheRoot, bucket, key string) (string, error) {
|
||||||
|
root := filepath.Clean(strings.TrimSpace(cacheRoot))
|
||||||
|
if root == "." || root == "" {
|
||||||
|
return "", fmt.Errorf("cache root is required")
|
||||||
|
}
|
||||||
|
bucket = strings.Trim(strings.TrimSpace(bucket), "/")
|
||||||
|
if bucket == "" || bucket == "." || bucket == ".." || strings.Contains(bucket, "/") || strings.Contains(bucket, `\`) {
|
||||||
|
return "", fmt.Errorf("bucket is required and must be a single path segment")
|
||||||
|
}
|
||||||
|
rawKey := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
|
||||||
|
if strings.HasPrefix(rawKey, "/") {
|
||||||
|
return "", fmt.Errorf("s3 key must not be absolute")
|
||||||
|
}
|
||||||
|
cleanKey := cleanCacheS3Key(rawKey)
|
||||||
|
if cleanKey == "" {
|
||||||
|
return "", fmt.Errorf("s3 key is required")
|
||||||
|
}
|
||||||
|
if cleanKey == ".." || strings.HasPrefix(cleanKey, "../") || strings.HasPrefix(cleanKey, "/") {
|
||||||
|
return "", fmt.Errorf("s3 key must not escape cache root")
|
||||||
|
}
|
||||||
|
return filepath.Join(root, "s3", bucket, filepath.FromSlash(cleanKey)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3AudioCacheNamespaceDir returns the durable local cache namespace for all
|
||||||
|
// Narratio S3 audio objects under one bucket/root prefix.
|
||||||
|
func S3AudioCacheNamespaceDir(cacheRoot, bucket, rootPrefix string) (string, error) {
|
||||||
|
sentinelKey := path.Join(cleanS3PathPart(rootPrefix), config.S3CampaignsSegment, ".narratio-cache-sentinel")
|
||||||
|
sentinelPath, err := S3AudioCachePath(cacheRoot, bucket, sentinelKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Dir(sentinelPath), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanCacheS3Key(key string) string {
|
||||||
|
normalized := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
|
||||||
|
normalized = strings.Trim(normalized, "/")
|
||||||
|
if normalized == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return path.Clean(normalized)
|
||||||
|
}
|
||||||
|
|
||||||
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
|
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
|
||||||
func SessionPreviousDir(paths SessionPaths) string {
|
func SessionPreviousDir(paths SessionPaths) string {
|
||||||
return paths.PreviousDir
|
return paths.PreviousDir
|
||||||
@@ -105,7 +162,19 @@ func SessionPreviousArtifactsDir(paths SessionPaths) string {
|
|||||||
|
|
||||||
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
|
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
|
||||||
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
|
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
|
||||||
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(artifactRelativePath))
|
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousArtifactCacheRelativePath(artifactRelativePath string) string {
|
||||||
|
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimSpace(artifactRelativePath))))
|
||||||
|
if rel == "." {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
const artifactsPrefix = "artifacts/"
|
||||||
|
if strings.HasPrefix(rel, artifactsPrefix) && len(rel) > len(artifactsPrefix) {
|
||||||
|
return strings.TrimPrefix(rel, artifactsPrefix)
|
||||||
|
}
|
||||||
|
return rel
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
||||||
|
|||||||
@@ -74,6 +74,11 @@ func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
|||||||
if artifactPath != wantArtifactPath {
|
if artifactPath != wantArtifactPath {
|
||||||
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
|
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
archiveRelativeArtifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "artifacts/session_recap.md")
|
||||||
|
if archiveRelativeArtifactPath != wantArtifactPath {
|
||||||
|
t.Fatalf("SessionPreviousArtifactPathForCampaign(archive-relative) = %q, want %q", archiveRelativeArtifactPath, wantArtifactPath)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
||||||
@@ -93,6 +98,12 @@ func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
|||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
|
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
got = SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
want = filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionPreviousArtifactPath(archive-relative) = %q, want %q", got, want)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSessionSpoolAudioDir(t *testing.T) {
|
func TestSessionSpoolAudioDir(t *testing.T) {
|
||||||
@@ -103,3 +114,66 @@ func TestSessionSpoolAudioDir(t *testing.T) {
|
|||||||
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
|
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSessionSpoolDir(t *testing.T) {
|
||||||
|
root := "/var/spool/narratio"
|
||||||
|
got := SessionSpoolDir(root, "forsaken", "2026-04-19")
|
||||||
|
want := filepath.Join(root, "forsaken", "2026-04-19")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionSpoolDir() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionSpoolRestoreAudioDir(t *testing.T) {
|
||||||
|
root := "/var/spool/narratio"
|
||||||
|
got := SessionSpoolRestoreAudioDir(root, "forsaken", "2026-04-19")
|
||||||
|
want := filepath.Join(root, "forsaken", "2026-04-19", "restore", "audio")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionSpoolRestoreAudioDir() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3AudioCachePath(t *testing.T) {
|
||||||
|
got, err := S3AudioCachePath("/var/cache/narratio", "my-dnd-archive", "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns", "forsaken", "sessions", "2026-04-19", "audio", "alice.flac")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("S3AudioCachePath() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3AudioCacheNamespaceDir(t *testing.T) {
|
||||||
|
got, err := S3AudioCacheNamespaceDir("/var/cache/narratio", "my-dnd-archive", "dnd")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCacheNamespaceDir() error = %v", err)
|
||||||
|
}
|
||||||
|
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("S3AudioCacheNamespaceDir() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3AudioCachePathRejectsUnsafeInputs(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
root string
|
||||||
|
bucket string
|
||||||
|
key string
|
||||||
|
}{
|
||||||
|
{name: "empty root", root: "", bucket: "bucket", key: "audio/a.flac"},
|
||||||
|
{name: "empty bucket", root: "/cache", bucket: "", key: "audio/a.flac"},
|
||||||
|
{name: "bucket slash", root: "/cache", bucket: "bad/bucket", key: "audio/a.flac"},
|
||||||
|
{name: "empty key", root: "/cache", bucket: "bucket", key: ""},
|
||||||
|
{name: "escaping key", root: "/cache", bucket: "bucket", key: "../audio/a.flac"},
|
||||||
|
{name: "absolute key", root: "/cache", bucket: "bucket", key: "/audio/a.flac"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got, err := S3AudioCachePath(tt.root, tt.bucket, tt.key); err == nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() = %q, want error", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
|
|||||||
return ensureS3TrailingSlash(key)
|
return ensureS3TrailingSlash(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// S3SessionConfigKey returns the session config key.
|
||||||
|
// Format: {session_prefix}/session.yml
|
||||||
|
func S3SessionConfigKey(sessionPrefix string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "session.yml")
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3SessionLocksKey returns the mutable session lock store key.
|
||||||
|
// Format: {session_prefix}/locks.yml
|
||||||
|
func S3SessionLocksKey(sessionPrefix string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "locks.yml")
|
||||||
|
}
|
||||||
|
|
||||||
// S3CurrentManifestKey returns the current manifest pointer key.
|
// S3CurrentManifestKey returns the current manifest pointer key.
|
||||||
// Format: {session_prefix}/current/manifest.json
|
// Format: {session_prefix}/current/manifest.json
|
||||||
func S3CurrentManifestKey(sessionPrefix string) string {
|
func S3CurrentManifestKey(sessionPrefix string) string {
|
||||||
|
|||||||
@@ -17,6 +17,16 @@ func TestS3KeyConstruction(t *testing.T) {
|
|||||||
t.Fatalf("audioPrefix = %q", audioPrefix)
|
t.Fatalf("audioPrefix = %q", audioPrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sessionConfigKey := S3SessionConfigKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
|
||||||
|
if sessionConfigKey != "dnd/campaigns/forsaken/sessions/2026-04-19/session.yml" {
|
||||||
|
t.Fatalf("session config key = %q", sessionConfigKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
locksKey := S3SessionLocksKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
|
||||||
|
if locksKey != "dnd/campaigns/forsaken/sessions/2026-04-19/locks.yml" {
|
||||||
|
t.Fatalf("locks key = %q", locksKey)
|
||||||
|
}
|
||||||
|
|
||||||
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
||||||
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
||||||
if runPrefix != wantRunPrefix {
|
if runPrefix != wantRunPrefix {
|
||||||
|
|||||||
223
internal/audio/s3_audio.go
Normal file
223
internal/audio/s3_audio.go
Normal file
@@ -0,0 +1,223 @@
|
|||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// S3MaterializeRequest describes one S3-backed audio materialization.
|
||||||
|
type S3MaterializeRequest struct {
|
||||||
|
Store storage.ObjectStore
|
||||||
|
Object storage.ObjectInfo
|
||||||
|
Bucket string
|
||||||
|
CacheRoot string
|
||||||
|
CacheEnabled bool
|
||||||
|
SpoolPath string
|
||||||
|
DestPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3MaterializeResult captures local materialization provenance.
|
||||||
|
type S3MaterializeResult struct {
|
||||||
|
Checksum string
|
||||||
|
CachePath string
|
||||||
|
SpoolPath string
|
||||||
|
CacheHit bool
|
||||||
|
Downloaded bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaterializeS3Audio installs one S3 audio object into the destination path,
|
||||||
|
// reusing and refreshing the durable local cache when enabled.
|
||||||
|
func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3MaterializeResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return S3MaterializeResult{}, err
|
||||||
|
}
|
||||||
|
if req.Store == nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("object store is required")
|
||||||
|
}
|
||||||
|
key := strings.TrimSpace(req.Object.Key)
|
||||||
|
if key == "" {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("s3 object key is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.DestPath) == "" {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("destination path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
result := S3MaterializeResult{}
|
||||||
|
if req.CacheEnabled {
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("resolve audio cache path: %w", err)
|
||||||
|
}
|
||||||
|
result.CachePath = cachePath
|
||||||
|
if ok, err := validCachedAudio(cachePath, req.Object.Size); err != nil {
|
||||||
|
return S3MaterializeResult{}, err
|
||||||
|
} else if ok {
|
||||||
|
checksum, err := copyFileAtomicWithChecksum(cachePath, req.DestPath, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("materialize cached audio %q: %w", cachePath, err)
|
||||||
|
}
|
||||||
|
result.Checksum = checksum
|
||||||
|
result.CacheHit = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spoolPath := strings.TrimSpace(req.SpoolPath)
|
||||||
|
if spoolPath == "" {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("spool path is required for s3 audio download")
|
||||||
|
}
|
||||||
|
if err := downloadObjectAtomic(ctx, req.Store, key, spoolPath); err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("download s3 audio object %q: %w", key, err)
|
||||||
|
}
|
||||||
|
if err := validateLocalAudio(spoolPath, req.Object.Size); err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("validate downloaded audio %q: %w", spoolPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
checksum, err := copyFileAtomicWithChecksum(spoolPath, req.DestPath, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("materialize downloaded audio %q: %w", filepath.Base(req.DestPath), err)
|
||||||
|
}
|
||||||
|
result.Checksum = checksum
|
||||||
|
result.SpoolPath = spoolPath
|
||||||
|
result.Downloaded = true
|
||||||
|
|
||||||
|
if result.CachePath != "" {
|
||||||
|
if _, err := copyFileAtomicWithChecksum(spoolPath, result.CachePath, 0o644); err != nil {
|
||||||
|
return S3MaterializeResult{}, fmt.Errorf("populate audio cache %q: %w", result.CachePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validCachedAudio(path string, expectedSize int64) (bool, error) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("stat cached audio %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return false, fmt.Errorf("cached audio path is a directory: %q", path)
|
||||||
|
}
|
||||||
|
if info.Size() <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if expectedSize > 0 && info.Size() != expectedSize {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLocalAudio(path string, expectedSize int64) error {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("stat: %w", err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return fmt.Errorf("path is a directory")
|
||||||
|
}
|
||||||
|
if info.Size() <= 0 {
|
||||||
|
return fmt.Errorf("file is empty")
|
||||||
|
}
|
||||||
|
if expectedSize > 0 && info.Size() != expectedSize {
|
||||||
|
return fmt.Errorf("size %d does not match remote size %d", info.Size(), expectedSize)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadObjectAtomic(ctx context.Context, store storage.ObjectStore, key, destPath string) error {
|
||||||
|
if strings.TrimSpace(destPath) == "" {
|
||||||
|
return fmt.Errorf("destination path is required")
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(destPath)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create destination directory: %w", err)
|
||||||
|
}
|
||||||
|
base := filepath.Base(destPath)
|
||||||
|
tmp, err := os.CreateTemp(dir, "."+base+".download-*.tmp")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("close temp file: %w", err)
|
||||||
|
}
|
||||||
|
removeTmp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTmp {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err := store.Download(ctx, key, tmpPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tmpPath, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("set temp file permissions: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpPath, destPath); err != nil {
|
||||||
|
return fmt.Errorf("install downloaded file: %w", err)
|
||||||
|
}
|
||||||
|
removeTmp = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFileAtomicWithChecksum(src, dst string, perm os.FileMode) (string, error) {
|
||||||
|
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
|
||||||
|
return "", fmt.Errorf("source and destination paths are required")
|
||||||
|
}
|
||||||
|
in, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer func() { _ = in.Close() }()
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||||
|
return "", fmt.Errorf("create destination directory: %w", err)
|
||||||
|
}
|
||||||
|
base := filepath.Base(dst)
|
||||||
|
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+base+".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
removeTmp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTmp {
|
||||||
|
_ = os.Remove(tmpPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
digest := sha256.New()
|
||||||
|
if _, err := io.Copy(io.MultiWriter(tmp, digest), in); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return "", fmt.Errorf("copy file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return "", fmt.Errorf("sync temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("close temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tmpPath, perm); err != nil {
|
||||||
|
return "", fmt.Errorf("chmod temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpPath, dst); err != nil {
|
||||||
|
return "", fmt.Errorf("install temp file: %w", err)
|
||||||
|
}
|
||||||
|
removeTmp = false
|
||||||
|
return hex.EncodeToString(digest.Sum(nil)), nil
|
||||||
|
}
|
||||||
145
internal/audio/s3_audio_test.go
Normal file
145
internal/audio/s3_audio_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
package audio
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMaterializeS3AudioCacheMissDownloadsAndPopulatesCache(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
|
||||||
|
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||||
|
|
||||||
|
result, err := MaterializeS3Audio(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||||
|
}
|
||||||
|
if !result.Downloaded || result.CacheHit {
|
||||||
|
t.Fatalf("result = %#v, want downloaded miss", result)
|
||||||
|
}
|
||||||
|
if len(fake.Downloads) != 1 {
|
||||||
|
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||||
|
}
|
||||||
|
assertFileEquals(t, req.DestPath, "audio")
|
||||||
|
assertFileEquals(t, req.SpoolPath, "audio")
|
||||||
|
assertFileEquals(t, result.CachePath, "audio")
|
||||||
|
if result.Checksum == "" {
|
||||||
|
t.Fatalf("checksum is empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||||
|
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cache path: %v", err)
|
||||||
|
}
|
||||||
|
writeAudioTestFile(t, cachePath, "audio")
|
||||||
|
|
||||||
|
result, err := MaterializeS3Audio(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||||
|
}
|
||||||
|
if !result.CacheHit || result.Downloaded {
|
||||||
|
t.Fatalf("result = %#v, want cache hit", result)
|
||||||
|
}
|
||||||
|
if len(fake.Downloads) != 0 {
|
||||||
|
t.Fatalf("downloads = %d, want 0", len(fake.Downloads))
|
||||||
|
}
|
||||||
|
assertFileEquals(t, req.DestPath, "audio")
|
||||||
|
if result.SpoolPath != "" {
|
||||||
|
t.Fatalf("spool path = %q, want empty on cache hit", result.SpoolPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterializeS3AudioInvalidCacheRefreshesFromS3(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio")})
|
||||||
|
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cache path: %v", err)
|
||||||
|
}
|
||||||
|
writeAudioTestFile(t, cachePath, "stale")
|
||||||
|
|
||||||
|
result, err := MaterializeS3Audio(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||||
|
}
|
||||||
|
if !result.Downloaded || result.CacheHit {
|
||||||
|
t.Fatalf("result = %#v, want refreshed miss", result)
|
||||||
|
}
|
||||||
|
if len(fake.Downloads) != 1 {
|
||||||
|
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||||
|
}
|
||||||
|
assertFileEquals(t, req.DestPath, "fresh-audio")
|
||||||
|
assertFileEquals(t, cachePath, "fresh-audio")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterializeS3AudioDownloadFailureLeavesDestinationMissing(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
fake := &storage.FakeBackend{DownloadErr: os.ErrPermission}
|
||||||
|
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
|
||||||
|
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||||
|
|
||||||
|
_, err := MaterializeS3Audio(context.Background(), req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "download s3 audio object") {
|
||||||
|
t.Fatalf("error = %v, want download error", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, req.DestPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testMaterializeRequest(t *testing.T, root string, fake *storage.FakeBackend, key string, size int64) S3MaterializeRequest {
|
||||||
|
t.Helper()
|
||||||
|
return S3MaterializeRequest{
|
||||||
|
Store: fake,
|
||||||
|
Object: storage.ObjectInfo{Key: key, Size: size, ETag: "etag"},
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
CacheRoot: filepath.Join(root, "cache"),
|
||||||
|
CacheEnabled: true,
|
||||||
|
SpoolPath: filepath.Join(root, "spool", "alice.flac"),
|
||||||
|
DestPath: filepath.Join(root, "work", "audio", "alice.flac"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAudioTestFile(t *testing.T, path, contents string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %q: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||||
|
t.Fatalf("write %q: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFileEquals(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %q: %v", path, err)
|
||||||
|
}
|
||||||
|
if string(data) != want {
|
||||||
|
t.Fatalf("%q = %q, want %q", path, string(data), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertMissing(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("stat %q = %v, want not exists", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
67
internal/config/cache_config_test.go
Normal file
67
internal/config/cache_config_test.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCacheDefaults(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`, `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
`)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Cache.Root != DefaultCacheRoot {
|
||||||
|
t.Fatalf("cache.root = %q, want %q", cfg.Pipeline.Cache.Root, DefaultCacheRoot)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio != DefaultCacheS3Audio {
|
||||||
|
t.Fatalf("cache.s3_audio = %v, want %v", cfg.Pipeline.Cache.S3Audio, DefaultCacheS3Audio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCacheStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
cache:
|
||||||
|
root: /var/cache/narratio
|
||||||
|
unknown: true
|
||||||
|
`, `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
`)
|
||||||
|
_, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("Load() error = %v, want strict decode failed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCacheValidationRequiresRootWhenS3AudioEnabled(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
cache:
|
||||||
|
root: " "
|
||||||
|
s3_audio: true
|
||||||
|
`, `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
`)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pipeline.cache.root is required") {
|
||||||
|
t.Fatalf("Validate() error = %v, want cache root error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
137
internal/config/campaign_config_test.go
Normal file
137
internal/config/campaign_config_test.go
Normal file
@@ -0,0 +1,137 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCampaignConfigDefaultSearchOrder(t *testing.T) {
|
||||||
|
want := []string{
|
||||||
|
"/usr/local/etc/narratio/campaign.yml",
|
||||||
|
"/etc/narratio/campaign.yml",
|
||||||
|
}
|
||||||
|
if len(DefaultCampaignConfigSearchPaths) != len(want) {
|
||||||
|
t.Fatalf("DefaultCampaignConfigSearchPaths = %#v, want %#v", DefaultCampaignConfigSearchPaths, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if DefaultCampaignConfigSearchPaths[i] != want[i] {
|
||||||
|
t.Fatalf("DefaultCampaignConfigSearchPaths[%d] = %q, want %q", i, DefaultCampaignConfigSearchPaths[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\nunknown: true\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMergeFillsStableInputs(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Session.Campaign != "sample-campaign" {
|
||||||
|
t.Fatalf("session campaign = %q, want campaign config value", cfg.Session.Campaign)
|
||||||
|
}
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./campaign-speakers.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMergeSessionOverridesStableInputs(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n speakers_file: ./session-speakers.yml\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./session-speakers.yml", sessionPath, "session_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMismatchFails(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ncampaign: other-campaign\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "does not match campaign config") {
|
||||||
|
t.Fatalf("error = %q, want campaign mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadMissingCampaignFileFails(t *testing.T) {
|
||||||
|
pipelinePath, _, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
missingCampaignPath := filepath.Join(filepath.Dir(sessionPath), "missing-campaign.yml")
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, missingCampaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "load campaign config") {
|
||||||
|
t.Fatalf("error = %q, want campaign load context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCampaignConfigTestFiles(t *testing.T, campaignYAML, sessionYAML string) (string, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := "workspace:\n root: " + filepath.ToSlash(filepath.Join(dir, "work")) + "\nwhisperx:\n transcribe_url: https://example.com/transcribe\nanalyzer:\n timeout: 20m\nnotification:\n timeout: 10s\n"
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertResolvedStableInput(t *testing.T, got ResolvedInputFile, wantPath, wantConfigPath, wantSource string) {
|
||||||
|
t.Helper()
|
||||||
|
if got.Path != wantPath || got.ConfigPath != wantConfigPath || got.Source != wantSource {
|
||||||
|
t.Fatalf("resolved input = %#v, want path=%q config_path=%q source=%q", got, wantPath, wantConfigPath, wantSource)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,11 +1,17 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Config is the resolved combined configuration from pipeline.yml and session.yml.
|
// Config is the resolved combined configuration from pipeline.yml,
|
||||||
|
// campaign.yml, and session.yml.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Pipeline *PipelineConfig
|
Pipeline *PipelineConfig
|
||||||
|
Campaign *CampaignConfig
|
||||||
Session *SessionConfig
|
Session *SessionConfig
|
||||||
PipelinePath string
|
PipelinePath string
|
||||||
|
CampaignPath string
|
||||||
SessionPath string
|
SessionPath string
|
||||||
|
|
||||||
|
StableInputs ResolvedStableInputs
|
||||||
|
SessionSource SessionSource
|
||||||
}
|
}
|
||||||
|
|
||||||
// PipelineConfig contains durable pipeline-level settings.
|
// PipelineConfig contains durable pipeline-level settings.
|
||||||
@@ -13,6 +19,7 @@ type PipelineConfig struct {
|
|||||||
Workspace WorkspaceConfig `yaml:"workspace"`
|
Workspace WorkspaceConfig `yaml:"workspace"`
|
||||||
Storage StorageConfig `yaml:"storage"`
|
Storage StorageConfig `yaml:"storage"`
|
||||||
Spool SpoolConfig `yaml:"spool"`
|
Spool SpoolConfig `yaml:"spool"`
|
||||||
|
Cache CacheConfig `yaml:"cache"`
|
||||||
Archive *ArchiveConfig `yaml:"archive"`
|
Archive *ArchiveConfig `yaml:"archive"`
|
||||||
Secrets *SecretsConfig `yaml:"secrets"`
|
Secrets *SecretsConfig `yaml:"secrets"`
|
||||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||||
@@ -25,6 +32,19 @@ type PipelineConfig struct {
|
|||||||
Notification NotificationConfig `yaml:"notification"`
|
Notification NotificationConfig `yaml:"notification"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CampaignConfig contains stable campaign-level identity and input defaults.
|
||||||
|
type CampaignConfig struct {
|
||||||
|
Campaign string `yaml:"campaign"`
|
||||||
|
Inputs CampaignInputsConfig `yaml:"inputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CampaignInputsConfig contains stable campaign-level input file references.
|
||||||
|
type CampaignInputsConfig struct {
|
||||||
|
SpeakersFile string `yaml:"speakers_file"`
|
||||||
|
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||||
|
GlossaryFile string `yaml:"glossary_file"`
|
||||||
|
}
|
||||||
|
|
||||||
// SessionConfig contains per-session inputs and metadata.
|
// SessionConfig contains per-session inputs and metadata.
|
||||||
type SessionConfig struct {
|
type SessionConfig struct {
|
||||||
SessionID string `yaml:"session_id"`
|
SessionID string `yaml:"session_id"`
|
||||||
@@ -71,11 +91,18 @@ type SpoolConfig struct {
|
|||||||
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CacheConfig configures durable local caches for reusable remote inputs.
|
||||||
|
type CacheConfig struct {
|
||||||
|
Root string `yaml:"root"`
|
||||||
|
S3Audio *bool `yaml:"s3_audio"`
|
||||||
|
}
|
||||||
|
|
||||||
// ArchiveConfig configures archive behavior and artifact promotions.
|
// ArchiveConfig configures archive behavior and artifact promotions.
|
||||||
type ArchiveConfig struct {
|
type ArchiveConfig struct {
|
||||||
Enabled *bool `yaml:"enabled"`
|
Enabled *bool `yaml:"enabled"`
|
||||||
UploadRun *bool `yaml:"upload_run"`
|
UploadRun *bool `yaml:"upload_run"`
|
||||||
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
||||||
|
Locks []ArchiveLockRule `yaml:"locks"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchivePromotionRule configures one artifact promotion mapping.
|
// ArchivePromotionRule configures one artifact promotion mapping.
|
||||||
@@ -85,6 +112,18 @@ type ArchivePromotionRule struct {
|
|||||||
Required *bool `yaml:"required"`
|
Required *bool `yaml:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ArchiveLockRule prevents one source-based promotion from overwriting its
|
||||||
|
// top-level archive destination.
|
||||||
|
type ArchiveLockRule struct {
|
||||||
|
Source string `yaml:"source"`
|
||||||
|
Reason string `yaml:"reason"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArchiveLockStore is the mutable per-session remote lock store.
|
||||||
|
type ArchiveLockStore struct {
|
||||||
|
Locks []ArchiveLockRule `yaml:"locks"`
|
||||||
|
}
|
||||||
|
|
||||||
// WhisperXConfig configures WhisperX adapter settings.
|
// WhisperXConfig configures WhisperX adapter settings.
|
||||||
type WhisperXConfig struct {
|
type WhisperXConfig struct {
|
||||||
TranscribeURL string `yaml:"transcribe_url"`
|
TranscribeURL string `yaml:"transcribe_url"`
|
||||||
@@ -229,3 +268,29 @@ type SessionInputsConfig struct {
|
|||||||
type SessionAudioS3Input struct {
|
type SessionAudioS3Input struct {
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolvedStableInputs records where stable input file paths came from after
|
||||||
|
// campaign/session merge.
|
||||||
|
type ResolvedStableInputs struct {
|
||||||
|
SpeakersFile ResolvedInputFile
|
||||||
|
AutocorrectFile ResolvedInputFile
|
||||||
|
GlossaryFile ResolvedInputFile
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvedInputFile records one merged config path and its source config file.
|
||||||
|
type ResolvedInputFile struct {
|
||||||
|
Path string
|
||||||
|
ConfigPath string
|
||||||
|
Source string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionSource records where session.yml came from before materialization.
|
||||||
|
type SessionSource struct {
|
||||||
|
Source string
|
||||||
|
LocalPath string
|
||||||
|
S3Bucket string
|
||||||
|
S3Key string
|
||||||
|
S3Size int64
|
||||||
|
S3ETag string
|
||||||
|
SpoolPath string
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Default filesystem locations for pipeline configuration lookup when --config
|
// Default filesystem locations for config lookup when config path flags are
|
||||||
// is omitted. Order is highest to lowest precedence.
|
// omitted. Order is highest to lowest precedence.
|
||||||
const (
|
const (
|
||||||
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
||||||
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
||||||
DefaultSessionConfigPathLocal = "./session.yml"
|
DefaultCampaignConfigPathUsrLocal = "/usr/local/etc/narratio/campaign.yml"
|
||||||
|
DefaultCampaignConfigPathEtc = "/etc/narratio/campaign.yml"
|
||||||
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
||||||
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
||||||
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
|
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
|
||||||
@@ -13,6 +14,8 @@ const (
|
|||||||
DefaultStorageS3RootPrefix = "dnd"
|
DefaultStorageS3RootPrefix = "dnd"
|
||||||
DefaultWorkspaceRoot = "/var/lib/narratio"
|
DefaultWorkspaceRoot = "/var/lib/narratio"
|
||||||
DefaultSpoolRoot = "/var/spool/narratio"
|
DefaultSpoolRoot = "/var/spool/narratio"
|
||||||
|
DefaultCacheRoot = "/var/cache/narratio"
|
||||||
|
DefaultCacheS3Audio = true
|
||||||
|
|
||||||
DefaultWhisperXLanguage = "en"
|
DefaultWhisperXLanguage = "en"
|
||||||
DefaultWhisperXTimeout = "30m"
|
DefaultWhisperXTimeout = "30m"
|
||||||
@@ -88,13 +91,22 @@ var DefaultPipelineConfigSearchPaths = []string{
|
|||||||
DefaultPipelineConfigPathEtc,
|
DefaultPipelineConfigPathEtc,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultCampaignConfigSearchPaths defines the default search order for
|
||||||
|
// campaign.yml when callers do not provide an explicit path.
|
||||||
|
//
|
||||||
|
// Keep this in a variable so future defaults can be extended without changing
|
||||||
|
// call sites.
|
||||||
|
var DefaultCampaignConfigSearchPaths = []string{
|
||||||
|
DefaultCampaignConfigPathUsrLocal,
|
||||||
|
DefaultCampaignConfigPathEtc,
|
||||||
|
}
|
||||||
|
|
||||||
// DefaultSessionConfigSearchPaths defines the default search order for
|
// DefaultSessionConfigSearchPaths defines the default search order for
|
||||||
// session.yml when callers do not provide an explicit path.
|
// session.yml when callers do not provide an explicit path.
|
||||||
//
|
//
|
||||||
// Keep this in a variable so future defaults can be extended without changing
|
// Keep this in a variable so future defaults can be extended without changing
|
||||||
// call sites.
|
// call sites.
|
||||||
var DefaultSessionConfigSearchPaths = []string{
|
var DefaultSessionConfigSearchPaths = []string{
|
||||||
DefaultSessionConfigPathLocal,
|
|
||||||
DefaultSessionConfigPathUsrLocal,
|
DefaultSessionConfigPathUsrLocal,
|
||||||
DefaultSessionConfigPathEtc,
|
DefaultSessionConfigPathEtc,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,15 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
|||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
||||||
|
func LoadCampaign(path string) (*CampaignConfig, error) {
|
||||||
|
var cfg CampaignConfig
|
||||||
|
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("load campaign config: %w", err)
|
||||||
|
}
|
||||||
|
return &cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
// LoadSession loads session configuration from a YAML file with strict field checking.
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
||||||
func LoadSession(path string) (*SessionConfig, error) {
|
func LoadSession(path string) (*SessionConfig, error) {
|
||||||
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
||||||
@@ -40,20 +49,25 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
||||||
}
|
}
|
||||||
|
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
|
||||||
|
}
|
||||||
|
|
||||||
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
|
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
|
||||||
|
// strict field checking after template rendering.
|
||||||
|
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
|
||||||
|
rendered, err := renderSessionTemplate(string(data), opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load session config: %w", err)
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg SessionConfig
|
var cfg SessionConfig
|
||||||
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
|
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
|
||||||
return nil, fmt.Errorf("load session config: %w", err)
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
||||||
path,
|
label,
|
||||||
strings.TrimSpace(opts.SessionID),
|
strings.TrimSpace(opts.SessionID),
|
||||||
strings.TrimSpace(cfg.SessionID),
|
strings.TrimSpace(cfg.SessionID),
|
||||||
)
|
)
|
||||||
@@ -63,7 +77,7 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
|||||||
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
||||||
path,
|
label,
|
||||||
strings.TrimSpace(opts.PreviousSessionID),
|
strings.TrimSpace(opts.PreviousSessionID),
|
||||||
strings.TrimSpace(cfg.PreviousSessionID),
|
strings.TrimSpace(cfg.PreviousSessionID),
|
||||||
)
|
)
|
||||||
@@ -71,32 +85,171 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
|||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load loads and resolves combined pipeline and session configuration.
|
// LoadArchiveLockStoreBytes loads a mutable session lock store with strict
|
||||||
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
// field checking and source validation.
|
||||||
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
|
func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
|
||||||
|
var store ArchiveLockStore
|
||||||
|
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
||||||
|
return nil, fmt.Errorf("load archive lock store: %w", err)
|
||||||
|
}
|
||||||
|
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load archive lock store: %w", err)
|
||||||
|
}
|
||||||
|
store.Locks = locks
|
||||||
|
return &store, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadWithSessionOptions loads and resolves combined pipeline and session
|
// MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
|
||||||
// configuration with session template options.
|
func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
|
||||||
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
if store == nil {
|
||||||
|
store = &ArchiveLockStore{}
|
||||||
|
}
|
||||||
|
data, err := yaml.Marshal(store)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("marshal archive lock store: %w", err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
||||||
|
// Passing only a session path is supported for package-internal compatibility;
|
||||||
|
// in that form campaign.yml is expected next to the session file.
|
||||||
|
func Load(pipelinePath string, paths ...string) (*Config, error) {
|
||||||
|
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
|
||||||
|
// session configuration with session template options.
|
||||||
|
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||||
pipelineCfg, err := LoadPipeline(pipelinePath)
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
campaignCfg, err := LoadCampaign(campaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
||||||
|
Source: "session_config",
|
||||||
|
LocalPath: sessionPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve builds final stage-facing configuration from already loaded
|
||||||
|
// pipeline, campaign, and session documents.
|
||||||
|
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
||||||
|
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionSource.Source) == "" {
|
||||||
|
sessionSource.Source = "session_config"
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
||||||
|
sessionSource.LocalPath = sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
return &Config{
|
return &Config{
|
||||||
Pipeline: pipelineCfg,
|
Pipeline: pipelineCfg,
|
||||||
Session: sessionCfg,
|
Campaign: campaignCfg,
|
||||||
PipelinePath: pipelinePath,
|
Session: sessionCfg,
|
||||||
SessionPath: sessionPath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
|
SessionPath: sessionPath,
|
||||||
|
StableInputs: stableInputs,
|
||||||
|
SessionSource: sessionSource,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
||||||
|
switch len(paths) {
|
||||||
|
case 1:
|
||||||
|
sessionPath = paths[0]
|
||||||
|
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
|
||||||
|
case 2:
|
||||||
|
campaignPath = paths[0]
|
||||||
|
sessionPath = paths[1]
|
||||||
|
default:
|
||||||
|
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
|
||||||
|
}
|
||||||
|
return campaignPath, sessionPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
|
||||||
|
if campaignCfg == nil {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
|
if sessionCfg == nil {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
campaignName := strings.TrimSpace(campaignCfg.Campaign)
|
||||||
|
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
|
||||||
|
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf(
|
||||||
|
"campaign/session config invalid: session campaign %q does not match campaign config %q",
|
||||||
|
sessionCampaign,
|
||||||
|
campaignName,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if sessionCampaign == "" {
|
||||||
|
sessionCfg.Campaign = campaignName
|
||||||
|
}
|
||||||
|
|
||||||
|
stable := ResolvedStableInputs{
|
||||||
|
SpeakersFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.SpeakersFile,
|
||||||
|
sessionCfg.Inputs.SpeakersFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
AutocorrectFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.AutocorrectFile,
|
||||||
|
sessionCfg.Inputs.AutocorrectFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
GlossaryFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.GlossaryFile,
|
||||||
|
sessionCfg.Inputs.GlossaryFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
|
||||||
|
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
|
||||||
|
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
|
||||||
|
return stable, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
|
||||||
|
if strings.TrimSpace(sessionValue) != "" {
|
||||||
|
return ResolvedInputFile{
|
||||||
|
Path: sessionValue,
|
||||||
|
ConfigPath: sessionPath,
|
||||||
|
Source: "session_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ResolvedInputFile{
|
||||||
|
Path: campaignValue,
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func decodeStrictYAML(kind, path string, out any) error {
|
func decodeStrictYAML(kind, path string, out any) error {
|
||||||
f, err := os.Open(path)
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -208,6 +361,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
|||||||
applyWorkspaceDefaults(&cfg.Workspace)
|
applyWorkspaceDefaults(&cfg.Workspace)
|
||||||
applyStorageDefaults(&cfg.Storage)
|
applyStorageDefaults(&cfg.Storage)
|
||||||
applySpoolDefaults(&cfg.Spool)
|
applySpoolDefaults(&cfg.Spool)
|
||||||
|
applyCacheDefaults(&cfg.Cache)
|
||||||
applyArchiveDefaults(&cfg.Archive)
|
applyArchiveDefaults(&cfg.Archive)
|
||||||
applyWhisperXDefaults(&cfg.WhisperX)
|
applyWhisperXDefaults(&cfg.WhisperX)
|
||||||
applySeriatimDefaults(&cfg.Seriatim)
|
applySeriatimDefaults(&cfg.Seriatim)
|
||||||
@@ -256,6 +410,18 @@ func applySpoolDefaults(cfg *SpoolConfig) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyCacheDefaults(cfg *CacheConfig) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cfg.Root == "" {
|
||||||
|
cfg.Root = DefaultCacheRoot
|
||||||
|
}
|
||||||
|
if cfg.S3Audio == nil {
|
||||||
|
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -904,6 +904,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
Report: boolPtr(true),
|
Report: boolPtr(true),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Campaign: &CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Session: &SessionConfig{
|
Session: &SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
@@ -963,6 +964,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
|
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
|
||||||
|
campaignPath := filepath.Join(examplesDir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
|
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -972,7 +974,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
|
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
|
||||||
cfg, err = Load(pipelinePath, sessionPath)
|
cfg, err = Load(pipelinePath, sessionPath)
|
||||||
} else {
|
} else {
|
||||||
cfg, err = LoadWithSessionOptions(pipelinePath, sessionPath, tt.sessionOpts)
|
cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, tt.sessionOpts)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load example config error = %v", err)
|
t.Fatalf("load example config error = %v", err)
|
||||||
@@ -1001,14 +1003,34 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline.yml: %v", err)
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
}
|
}
|
||||||
|
campaignYAML := `campaign: ` + campaignNameFromSessionYAML(sessionYAML) + `
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write session.yml: %v", err)
|
t.Fatalf("write session.yml: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, sessionPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func campaignNameFromSessionYAML(sessionYAML string) string {
|
||||||
|
for _, line := range strings.Split(sessionYAML, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(trimmed, "campaign:") {
|
||||||
|
return strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "campaign:")), `"'`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "sample-campaign"
|
||||||
|
}
|
||||||
|
|||||||
@@ -241,3 +241,38 @@ inputs:
|
|||||||
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionBytesWithOptionsUsesSameTemplateAndStrictDecode(t *testing.T) {
|
||||||
|
sessionYAML := []byte(`session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", sessionYAML, SessionLoadOptions{SessionID: "2026-05-03"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionBytesWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SessionID != "2026-05-03" {
|
||||||
|
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\nunknown: true\n"), SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected strict decode error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("error = %q, want strict decode context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionBytesWithOptionsMismatchFails(t *testing.T) {
|
||||||
|
_, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\n"), SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected mismatch error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -287,6 +287,100 @@ archive:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockValidation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
pipelineYML string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid built in source",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
reason: reviewed transcript
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid configured source",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- reason: no source
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.archive.locks[0].source is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.unknown
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
- source: " narratio.transcript.trimmed "
|
||||||
|
`,
|
||||||
|
wantErr: "duplicates another archive lock source",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||||
|
_, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("Load() error = %v, want strict decode failed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
archive:
|
||||||
@@ -301,6 +395,54 @@ archive:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||||
|
store, err := LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
reason: reviewed
|
||||||
|
`), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.trimmed" || store.Locks[0].Reason != "reviewed" {
|
||||||
|
t.Fatalf("locks = %#v", store.Locks)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
`), nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
`), nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "duplicates another archive lock source") {
|
||||||
|
t.Fatalf("duplicate error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
|
||||||
|
merged := MergeArchiveLockRules(
|
||||||
|
[]ArchiveLockRule{{Source: "narratio.transcript.trimmed", Reason: "static"}},
|
||||||
|
[]ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.trimmed", Reason: "remote"},
|
||||||
|
{Source: "narratio.transcript.full", Reason: "remote full"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if len(merged) != 2 {
|
||||||
|
t.Fatalf("merged len = %d, want 2: %#v", len(merged), merged)
|
||||||
|
}
|
||||||
|
if merged[0].Source != "narratio.transcript.trimmed" || merged[0].Reason != "static" {
|
||||||
|
t.Fatalf("merged[0] = %#v, want static lock", merged[0])
|
||||||
|
}
|
||||||
|
if merged[1].Source != "narratio.transcript.full" {
|
||||||
|
t.Fatalf("merged[1] = %#v, want remote full lock", merged[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSessionAudioS3Validation(t *testing.T) {
|
func TestSessionAudioS3Validation(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ func Validate(cfg *Config) error {
|
|||||||
if cfg.Pipeline == nil {
|
if cfg.Pipeline == nil {
|
||||||
return fmt.Errorf("pipeline config is required")
|
return fmt.Errorf("pipeline config is required")
|
||||||
}
|
}
|
||||||
|
if cfg.Campaign == nil {
|
||||||
|
return fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
if cfg.Session == nil {
|
if cfg.Session == nil {
|
||||||
return fmt.Errorf("session config is required")
|
return fmt.Errorf("session config is required")
|
||||||
}
|
}
|
||||||
@@ -24,6 +27,9 @@ func Validate(cfg *Config) error {
|
|||||||
if err := validatePipeline(cfg.Pipeline); err != nil {
|
if err := validatePipeline(cfg.Pipeline); err != nil {
|
||||||
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
||||||
}
|
}
|
||||||
|
if err := validateCampaign(cfg.Campaign); err != nil {
|
||||||
|
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
|
||||||
|
}
|
||||||
if err := validateSession(cfg.Session); err != nil {
|
if err := validateSession(cfg.Session); err != nil {
|
||||||
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
||||||
}
|
}
|
||||||
@@ -34,6 +40,16 @@ func Validate(cfg *Config) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCampaign(cfg *CampaignConfig) error {
|
||||||
|
if cfg == nil {
|
||||||
|
return fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.Campaign) == "" {
|
||||||
|
return fmt.Errorf("campaign.campaign is required")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validatePipeline(cfg *PipelineConfig) error {
|
func validatePipeline(cfg *PipelineConfig) error {
|
||||||
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
||||||
return fmt.Errorf("pipeline.workspace.root is required")
|
return fmt.Errorf("pipeline.workspace.root is required")
|
||||||
@@ -47,6 +63,9 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
if err := validateSpool(cfg.Spool); err != nil {
|
if err := validateSpool(cfg.Spool); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := validateCache(cfg.Cache); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
|
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -104,6 +123,13 @@ func validateSpool(cfg SpoolConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCache(cfg CacheConfig) error {
|
||||||
|
if cfg.S3Audio != nil && *cfg.S3Audio && strings.TrimSpace(cfg.Root) == "" {
|
||||||
|
return fmt.Errorf("pipeline.cache.root is required when pipeline.cache.s3_audio is true")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -136,9 +162,69 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
seenDest[normalizedDest] = struct{}{}
|
seenDest[normalizedDest] = struct{}{}
|
||||||
}
|
}
|
||||||
|
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cfg.Locks = locks
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ValidateArchiveLockRules validates and normalizes source-based archive locks.
|
||||||
|
func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumConfig, label string) ([]ArchiveLockRule, error) {
|
||||||
|
seenLocks := map[string]struct{}{}
|
||||||
|
out := make([]ArchiveLockRule, 0, len(locks))
|
||||||
|
if strings.TrimSpace(label) == "" {
|
||||||
|
label = "archive.locks"
|
||||||
|
}
|
||||||
|
for i, item := range locks {
|
||||||
|
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||||
|
source := strings.TrimSpace(item.Source)
|
||||||
|
if source == "" {
|
||||||
|
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||||
|
}
|
||||||
|
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
|
}
|
||||||
|
if _, ok := seenLocks[source]; ok {
|
||||||
|
return nil, fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
|
||||||
|
}
|
||||||
|
seenLocks[source] = struct{}{}
|
||||||
|
out = append(out, ArchiveLockRule{
|
||||||
|
Source: source,
|
||||||
|
Reason: strings.TrimSpace(item.Reason),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeArchiveLockRules returns the union of static and remote locks. Static
|
||||||
|
// locks win when both sources contain the same lock.
|
||||||
|
func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []ArchiveLockRule {
|
||||||
|
out := make([]ArchiveLockRule, 0, len(staticLocks)+len(remoteLocks))
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, item := range staticLocks {
|
||||||
|
source := strings.TrimSpace(item.Source)
|
||||||
|
if source == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||||
|
seen[source] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, item := range remoteLocks {
|
||||||
|
source := strings.TrimSpace(item.Source)
|
||||||
|
if source == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[source]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||||
|
seen[source] = struct{}{}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
||||||
trimmed := strings.TrimSpace(source)
|
trimmed := strings.TrimSpace(source)
|
||||||
switch trimmed {
|
switch trimmed {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ type InputRecord struct {
|
|||||||
S3Size int64 `json:"s3_size,omitempty"`
|
S3Size int64 `json:"s3_size,omitempty"`
|
||||||
S3ETag string `json:"s3_etag,omitempty"`
|
S3ETag string `json:"s3_etag,omitempty"`
|
||||||
SpoolPath string `json:"spool_path,omitempty"`
|
SpoolPath string `json:"spool_path,omitempty"`
|
||||||
|
CachePath string `json:"cache_path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
||||||
|
|||||||
@@ -126,7 +126,14 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
||||||
}
|
}
|
||||||
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
|
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
|
||||||
|
sessionPaths,
|
||||||
|
m,
|
||||||
|
runtimeCatalog,
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts,
|
||||||
|
env.Config.Pipeline.Archive.Locks,
|
||||||
|
sessionPrefix,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
||||||
}
|
}
|
||||||
@@ -166,6 +173,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
promotedUploaded,
|
promotedUploaded,
|
||||||
previousUploaded,
|
previousUploaded,
|
||||||
skippedOptional,
|
skippedOptional,
|
||||||
|
lockedPromotions,
|
||||||
currentManifestKey,
|
currentManifestKey,
|
||||||
))
|
))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -204,6 +212,8 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
"previous_files_uploaded": len(previousUploaded),
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
"previous_uploaded_paths": previousUploaded,
|
"previous_uploaded_paths": previousUploaded,
|
||||||
"skipped_optional_promotions": skippedOptional,
|
"skipped_optional_promotions": skippedOptional,
|
||||||
|
"locked_promotion_count": len(lockedPromotions),
|
||||||
|
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": currentRunPointerKey,
|
"current_run_id_key": currentRunPointerKey,
|
||||||
"current_pointer_written": true,
|
"current_pointer_written": true,
|
||||||
@@ -220,6 +230,16 @@ type archivePromotion struct {
|
|||||||
Provenance string
|
Provenance string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type archiveLockedPromotion struct {
|
||||||
|
Source string
|
||||||
|
Dest string
|
||||||
|
RemoteKey string
|
||||||
|
Reason string
|
||||||
|
Required bool
|
||||||
|
LocalPath string
|
||||||
|
Provenance string
|
||||||
|
}
|
||||||
|
|
||||||
func archiveDisabled(env *Env) bool {
|
func archiveDisabled(env *Env) bool {
|
||||||
cfg := env.Config.Pipeline.Archive
|
cfg := env.Config.Pipeline.Archive
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
@@ -319,26 +339,53 @@ func resolveArchivePromotions(
|
|||||||
m *manifest.Manifest,
|
m *manifest.Manifest,
|
||||||
catalog *artifacts.ArtifactCatalog,
|
catalog *artifacts.ArtifactCatalog,
|
||||||
rules []config.ArchivePromotionRule,
|
rules []config.ArchivePromotionRule,
|
||||||
) ([]archivePromotion, []string, error) {
|
locks []config.ArchiveLockRule,
|
||||||
|
sessionPrefix string,
|
||||||
|
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
|
||||||
out := make([]archivePromotion, 0, len(rules))
|
out := make([]archivePromotion, 0, len(rules))
|
||||||
skippedOptional := make([]string, 0)
|
skippedOptional := make([]string, 0)
|
||||||
|
lockedPromotions := make([]archiveLockedPromotion, 0)
|
||||||
|
lockSet := archiveLockSet(locks)
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
required := rule.Required == nil || *rule.Required
|
required := rule.Required == nil || *rule.Required
|
||||||
dest, err := resolveArchivePromotionDest(rule, catalog)
|
dest, err := resolveArchivePromotionDest(rule, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("source %q: %w", source, err)
|
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||||
}
|
}
|
||||||
|
lock, locked := lockSet[source]
|
||||||
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if locked {
|
||||||
|
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||||
|
Source: source,
|
||||||
|
Dest: dest,
|
||||||
|
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||||
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
|
Required: required,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
||||||
skippedOptional = append(skippedOptional, dest)
|
skippedOptional = append(skippedOptional, dest)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||||
}
|
}
|
||||||
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||||
|
}
|
||||||
|
if locked {
|
||||||
|
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||||
|
Source: source,
|
||||||
|
Dest: dest,
|
||||||
|
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||||
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
|
Required: required,
|
||||||
|
LocalPath: resolved.Path,
|
||||||
|
Provenance: resolved.Provenance,
|
||||||
|
})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, archivePromotion{
|
out = append(out, archivePromotion{
|
||||||
Source: source,
|
Source: source,
|
||||||
@@ -348,7 +395,21 @@ func resolveArchivePromotions(
|
|||||||
Provenance: resolved.Provenance,
|
Provenance: resolved.Provenance,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out, skippedOptional, nil
|
return out, skippedOptional, lockedPromotions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||||
|
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||||
|
for _, lock := range locks {
|
||||||
|
source := strings.TrimSpace(lock.Source)
|
||||||
|
if source == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lock.Source = source
|
||||||
|
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||||
|
out[source] = lock
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
||||||
@@ -663,6 +724,7 @@ func archiveMetadataPreview(
|
|||||||
promotedUploaded []string,
|
promotedUploaded []string,
|
||||||
previousUploaded []string,
|
previousUploaded []string,
|
||||||
skippedOptional []string,
|
skippedOptional []string,
|
||||||
|
lockedPromotions []archiveLockedPromotion,
|
||||||
currentManifestKey string,
|
currentManifestKey string,
|
||||||
) map[string]any {
|
) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
@@ -677,9 +739,27 @@ func archiveMetadataPreview(
|
|||||||
"previous_files_uploaded": len(previousUploaded),
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
||||||
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
||||||
|
"locked_promotion_count": len(lockedPromotions),
|
||||||
|
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||||
"current_pointer_written": false,
|
"current_pointer_written": false,
|
||||||
"audio_upload_skipped": true,
|
"audio_upload_skipped": true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
|
||||||
|
out := make([]map[string]any, 0, len(locked))
|
||||||
|
for _, item := range locked {
|
||||||
|
out = append(out, map[string]any{
|
||||||
|
"source": item.Source,
|
||||||
|
"dest": item.Dest,
|
||||||
|
"remote_key": item.RemoteKey,
|
||||||
|
"reason": item.Reason,
|
||||||
|
"required": item.Required,
|
||||||
|
"local_path": item.LocalPath,
|
||||||
|
"provenance": item.Provenance,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package stage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -215,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
|
||||||
|
}
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||||
|
if _, ok := fake.Objects[trimmedKey]; ok {
|
||||||
|
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
|
||||||
|
}
|
||||||
|
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
|
||||||
|
if _, ok := fake.Objects[recapKey]; !ok {
|
||||||
|
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
|
||||||
|
}
|
||||||
|
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
|
||||||
|
if _, ok := fake.Objects[runTrimmedKey]; !ok {
|
||||||
|
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
|
||||||
|
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
|
||||||
|
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Metadata["promoted_files_uploaded"] != 1 {
|
||||||
|
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
|
||||||
|
}
|
||||||
|
if result.Metadata["locked_promotion_count"] != 1 {
|
||||||
|
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
|
||||||
|
}
|
||||||
|
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||||
|
if len(locked) != 1 {
|
||||||
|
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||||
|
}
|
||||||
|
if locked[0]["source"] != "narratio.transcript.trimmed" ||
|
||||||
|
locked[0]["dest"] != "transcripts/trimmed.json" ||
|
||||||
|
locked[0]["remote_key"] != trimmedKey ||
|
||||||
|
locked[0]["reason"] != "human reviewed" ||
|
||||||
|
locked[0]["required"] != true ||
|
||||||
|
locked[0]["local_path"] == "" ||
|
||||||
|
locked[0]["provenance"] == "" {
|
||||||
|
t.Fatalf("locked promotion metadata = %#v", locked[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
|
||||||
|
var current map[string]any
|
||||||
|
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, ¤t); err != nil {
|
||||||
|
t.Fatalf("unmarshal current manifest: %v", err)
|
||||||
|
}
|
||||||
|
stages := current["stages"].(map[string]any)
|
||||||
|
archive := stages["archive"].(map[string]any)
|
||||||
|
meta := archive["metadata"].(map[string]any)
|
||||||
|
if meta["locked_promotion_count"] != float64(1) {
|
||||||
|
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
|
||||||
|
}
|
||||||
|
items := meta["locked_promotions"].([]any)
|
||||||
|
if len(items) != 1 {
|
||||||
|
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
|
||||||
|
if _, ok := fake.Objects[mergedKey]; ok {
|
||||||
|
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
||||||
|
t.Fatalf("current run pointer should be written for locked missing promotion")
|
||||||
|
}
|
||||||
|
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||||
|
if len(locked) != 1 {
|
||||||
|
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||||
|
}
|
||||||
|
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
|
||||||
|
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.trimmed", Reason: "already published"},
|
||||||
|
}
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
|
||||||
|
|
||||||
|
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := string(fake.Objects[trimmedKey].Data)
|
||||||
|
if got != "previously published\n" {
|
||||||
|
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
|
||||||
|
}
|
||||||
|
for _, upload := range fake.Uploads {
|
||||||
|
if upload.Key == trimmedKey {
|
||||||
|
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
store := artifacts.NewLocalStore(root)
|
store := artifacts.NewLocalStore(root)
|
||||||
cfgDir := t.TempDir()
|
cfgDir := t.TempDir()
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
|
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||||
|
writeStageTestFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
|
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
|
||||||
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
@@ -48,7 +50,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
env := &Env{
|
env := &Env{
|
||||||
Config: &config.Config{
|
Config: &config.Config{
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Pipeline: &config.PipelineConfig{
|
Pipeline: &config.PipelineConfig{
|
||||||
Workspace: config.WorkspaceConfig{Root: root},
|
Workspace: config.WorkspaceConfig{Root: root},
|
||||||
Storage: config.StorageConfig{
|
Storage: config.StorageConfig{
|
||||||
@@ -62,14 +66,28 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
|
||||||
AutocorrectFile: "./autocorrect.yml",
|
|
||||||
GlossaryFile: "./glossary.yml",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/audio"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
@@ -25,6 +26,7 @@ func (prepareStage) Name() string { return "prepare" }
|
|||||||
func (prepareStage) Declares() IODecl {
|
func (prepareStage) Declares() IODecl {
|
||||||
return IODecl{
|
return IODecl{
|
||||||
Inputs: []artifacts.Ref{
|
Inputs: []artifacts.Ref{
|
||||||
|
{Kind: "config", Category: "inputs", RelativePath: "campaign.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
||||||
@@ -59,21 +61,30 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
campaignSrc := env.Config.CampaignPath
|
||||||
|
if err := requireFile(campaignSrc, "campaign.yml"); err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
sessionSrc := env.Config.SessionPath
|
sessionSrc := env.Config.SessionPath
|
||||||
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
||||||
return nil, fmt.Errorf("prepare: %w", err)
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
}
|
}
|
||||||
sessionDir := filepath.Dir(sessionSrc)
|
sessionDir := filepath.Dir(sessionSrc)
|
||||||
|
|
||||||
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile)
|
speakersInput := stableInputSource(env.Config.StableInputs.SpeakersFile, env.Config.Session.Inputs.SpeakersFile, sessionSrc)
|
||||||
|
autocorrectInput := stableInputSource(env.Config.StableInputs.AutocorrectFile, env.Config.Session.Inputs.AutocorrectFile, sessionSrc)
|
||||||
|
glossaryInput := stableInputSource(env.Config.StableInputs.GlossaryFile, env.Config.Session.Inputs.GlossaryFile, sessionSrc)
|
||||||
|
|
||||||
|
speakersSrc, err := resolveConfigRelativePath(speakersInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
||||||
}
|
}
|
||||||
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile)
|
autocorrectSrc, err := resolveConfigRelativePath(autocorrectInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
||||||
}
|
}
|
||||||
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile)
|
glossarySrc, err := resolveConfigRelativePath(glossaryInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
||||||
}
|
}
|
||||||
@@ -96,17 +107,44 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
|
inputs := make([]manifest.InputRecord, 0, 6+len(resolvedLocalAudio))
|
||||||
registerInput := func(kind, path, checksum string) {
|
registerInput := func(kind, path, checksum string) {
|
||||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
||||||
}
|
}
|
||||||
|
registerConfigInput := func(kind, path, checksum, source string) {
|
||||||
|
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum, Source: source})
|
||||||
|
}
|
||||||
|
registerSessionConfigInput := func(path, checksum string) {
|
||||||
|
source := env.Config.SessionSource
|
||||||
|
if strings.TrimSpace(source.Source) == "" {
|
||||||
|
source.Source = "session_config"
|
||||||
|
}
|
||||||
|
inputs = append(inputs, manifest.InputRecord{
|
||||||
|
Kind: "session_config",
|
||||||
|
Path: path,
|
||||||
|
Checksum: checksum,
|
||||||
|
Source: source.Source,
|
||||||
|
S3Bucket: source.S3Bucket,
|
||||||
|
S3Key: source.S3Key,
|
||||||
|
S3Size: source.S3Size,
|
||||||
|
S3ETag: source.S3ETag,
|
||||||
|
SpoolPath: source.SpoolPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
campaignDst := filepath.Join(paths.InputsDir, "campaign.yml")
|
||||||
|
campaignChecksum, err := copyFileIfChanged(env.ArtifactStore, campaignSrc, campaignDst)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: materialize campaign.yml: %w", err)
|
||||||
|
}
|
||||||
|
registerConfigInput("campaign_config", campaignDst, campaignChecksum, "campaign_config")
|
||||||
|
|
||||||
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
||||||
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
||||||
}
|
}
|
||||||
registerInput("session_config", sessionDst, sessionChecksum)
|
registerSessionConfigInput(sessionDst, sessionChecksum)
|
||||||
|
|
||||||
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -120,25 +158,29 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
|
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
|
||||||
|
|
||||||
for _, cfgFile := range []struct {
|
for _, cfgFile := range []struct {
|
||||||
kind string
|
kind string
|
||||||
src string
|
src string
|
||||||
dst string
|
dst string
|
||||||
|
source string
|
||||||
}{
|
}{
|
||||||
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")},
|
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml"), source: speakersInput.Source},
|
||||||
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")},
|
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml"), source: autocorrectInput.Source},
|
||||||
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")},
|
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml"), source: glossaryInput.Source},
|
||||||
} {
|
} {
|
||||||
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
||||||
}
|
}
|
||||||
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
registerConfigInput(cfgFile.kind, cfgFile.dst, checksum, cfgFile.source)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var audioCacheStats s3AudioMaterializationStats
|
||||||
if useS3Audio {
|
if useS3Audio {
|
||||||
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
|
stats, err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
|
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
|
||||||
}
|
}
|
||||||
|
audioCacheStats = stats
|
||||||
} else {
|
} else {
|
||||||
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
|
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
|
||||||
return nil, fmt.Errorf("prepare: %w", err)
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
@@ -173,6 +215,11 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
"inputs_count": len(inputs),
|
"inputs_count": len(inputs),
|
||||||
"audio_files_resolved": countAudioInputs(inputs),
|
"audio_files_resolved": countAudioInputs(inputs),
|
||||||
}
|
}
|
||||||
|
if useS3Audio {
|
||||||
|
metadata["audio_cache_hits"] = audioCacheStats.CacheHits
|
||||||
|
metadata["audio_cache_misses"] = audioCacheStats.CacheMisses
|
||||||
|
metadata["audio_s3_downloads"] = audioCacheStats.Downloads
|
||||||
|
}
|
||||||
if len(previousRequirements) > 0 {
|
if len(previousRequirements) > 0 {
|
||||||
metadata["previous_requirements_count"] = len(previousRequirements)
|
metadata["previous_requirements_count"] = len(previousRequirements)
|
||||||
if previousHydration != nil {
|
if previousHydration != nil {
|
||||||
@@ -195,6 +242,31 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
|
|||||||
return yaml.Marshal(cfg)
|
return yaml.Marshal(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stableInputSource(resolved config.ResolvedInputFile, fallbackPath, fallbackConfigPath string) config.ResolvedInputFile {
|
||||||
|
if strings.TrimSpace(resolved.Path) != "" || strings.TrimSpace(resolved.ConfigPath) != "" || strings.TrimSpace(resolved.Source) != "" {
|
||||||
|
if strings.TrimSpace(resolved.ConfigPath) == "" {
|
||||||
|
resolved.ConfigPath = fallbackConfigPath
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(resolved.Source) == "" {
|
||||||
|
resolved.Source = "session_config"
|
||||||
|
}
|
||||||
|
return resolved
|
||||||
|
}
|
||||||
|
return config.ResolvedInputFile{
|
||||||
|
Path: fallbackPath,
|
||||||
|
ConfigPath: fallbackConfigPath,
|
||||||
|
Source: "session_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||||
|
basePath := strings.TrimSpace(input.ConfigPath)
|
||||||
|
if basePath == "" {
|
||||||
|
return "", fmt.Errorf("source config path is required")
|
||||||
|
}
|
||||||
|
return resolvePath(filepath.Dir(basePath), input.Path)
|
||||||
|
}
|
||||||
|
|
||||||
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
||||||
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
||||||
if inputs.AudioS3 != nil {
|
if inputs.AudioS3 != nil {
|
||||||
@@ -282,28 +354,34 @@ func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolve
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
|
type s3AudioMaterializationStats struct {
|
||||||
|
CacheHits int
|
||||||
|
CacheMisses int
|
||||||
|
Downloads int
|
||||||
|
}
|
||||||
|
|
||||||
|
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) (s3AudioMaterializationStats, error) {
|
||||||
if env.ObjectStore == nil {
|
if env.ObjectStore == nil {
|
||||||
return fmt.Errorf("s3 audio input requires object store backend")
|
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires object store backend")
|
||||||
}
|
}
|
||||||
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
||||||
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
|
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
|
||||||
}
|
}
|
||||||
|
|
||||||
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
if campaign == "" {
|
if campaign == "" {
|
||||||
return fmt.Errorf("session campaign is required for s3 audio input")
|
return s3AudioMaterializationStats{}, fmt.Errorf("session campaign is required for s3 audio input")
|
||||||
}
|
}
|
||||||
runID := strings.TrimSpace(m.RunID)
|
runID := strings.TrimSpace(m.RunID)
|
||||||
if runID == "" {
|
if runID == "" {
|
||||||
return fmt.Errorf("run id is required for s3 audio input")
|
return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||||
@@ -321,7 +399,7 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
|||||||
return audioObjects[i].Key < audioObjects[j].Key
|
return audioObjects[i].Key < audioObjects[j].Key
|
||||||
})
|
})
|
||||||
if len(audioObjects) == 0 {
|
if len(audioObjects) == 0 {
|
||||||
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||||
@@ -331,45 +409,60 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
|||||||
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
|
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
|
||||||
|
|
||||||
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
|
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
|
||||||
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
|
return s3AudioMaterializationStats{}, fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
|
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
|
||||||
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
seenBase := map[string]string{}
|
seenBase := map[string]string{}
|
||||||
|
stats := s3AudioMaterializationStats{}
|
||||||
|
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
|
||||||
for _, obj := range audioObjects {
|
for _, obj := range audioObjects {
|
||||||
base := path.Base(obj.Key)
|
base := path.Base(obj.Key)
|
||||||
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
||||||
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||||
}
|
}
|
||||||
seenBase[base] = obj.Key
|
seenBase[base] = obj.Key
|
||||||
|
|
||||||
spoolPath := filepath.Join(spoolAudioDir, base)
|
spoolPath := filepath.Join(spoolAudioDir, base)
|
||||||
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
|
|
||||||
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
workPath := filepath.Join(workAudioDir, base)
|
workPath := filepath.Join(workAudioDir, base)
|
||||||
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
|
result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||||
|
Store: env.ObjectStore,
|
||||||
|
Object: obj,
|
||||||
|
Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
||||||
|
CacheRoot: strings.TrimSpace(env.Config.Pipeline.Cache.Root),
|
||||||
|
CacheEnabled: cacheEnabled,
|
||||||
|
SpoolPath: spoolPath,
|
||||||
|
DestPath: workPath,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
|
return s3AudioMaterializationStats{}, err
|
||||||
|
}
|
||||||
|
if result.CacheHit {
|
||||||
|
stats.CacheHits++
|
||||||
|
} else {
|
||||||
|
stats.CacheMisses++
|
||||||
|
}
|
||||||
|
if result.Downloaded {
|
||||||
|
stats.Downloads++
|
||||||
}
|
}
|
||||||
|
|
||||||
*inputs = append(*inputs, manifest.InputRecord{
|
*inputs = append(*inputs, manifest.InputRecord{
|
||||||
Kind: "audio",
|
Kind: "audio",
|
||||||
Path: workPath,
|
Path: workPath,
|
||||||
Checksum: checksum,
|
Checksum: result.Checksum,
|
||||||
Source: "s3",
|
Source: "s3",
|
||||||
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
||||||
S3Key: obj.Key,
|
S3Key: obj.Key,
|
||||||
S3Size: obj.Size,
|
S3Size: obj.Size,
|
||||||
S3ETag: obj.ETag,
|
S3ETag: obj.ETag,
|
||||||
SpoolPath: spoolPath,
|
SpoolPath: result.SpoolPath,
|
||||||
|
CachePath: result.CachePath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func countAudioInputs(inputs []manifest.InputRecord) int {
|
func countAudioInputs(inputs []manifest.InputRecord) int {
|
||||||
|
|||||||
@@ -50,6 +50,10 @@ func TestHydratePreviousSessionArtifactsDownloadsManifestAndRequiredArtifact(t *
|
|||||||
if _, err := os.Stat(recapPath); err != nil {
|
if _, err := os.Stat(recapPath); err != nil {
|
||||||
t.Fatalf("previous artifact missing: %v", err)
|
t.Fatalf("previous artifact missing: %v", err)
|
||||||
}
|
}
|
||||||
|
nestedRecapPath := filepath.Join(sessionPaths.PreviousArtifactsDir, "artifacts", "session_recap.md")
|
||||||
|
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
|
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
|
||||||
if manifestInput == nil {
|
if manifestInput == nil {
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
|||||||
|
|
||||||
paths := sessionPathsForEnv(env, m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
for _, p := range []string{
|
for _, p := range []string{
|
||||||
|
filepath.Join(paths.InputsDir, "campaign.yml"),
|
||||||
filepath.Join(paths.InputsDir, "session.yml"),
|
filepath.Join(paths.InputsDir, "session.yml"),
|
||||||
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
||||||
filepath.Join(paths.InputsDir, "speakers.yml"),
|
filepath.Join(paths.InputsDir, "speakers.yml"),
|
||||||
@@ -48,8 +49,8 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(m.Inputs) != 7 {
|
if len(m.Inputs) != 8 {
|
||||||
t.Fatalf("manifest inputs len = %d, want 7", len(m.Inputs))
|
t.Fatalf("manifest inputs len = %d, want 8", len(m.Inputs))
|
||||||
}
|
}
|
||||||
for _, in := range m.Inputs {
|
for _, in := range m.Inputs {
|
||||||
if in.Checksum == "" {
|
if in.Checksum == "" {
|
||||||
@@ -151,6 +152,64 @@ func TestPrepareStageIdempotent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRecordsLocalSessionProvenance(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||||
|
if sessionInput.Source != "session_config" {
|
||||||
|
t.Fatalf("session source = %q, want session_config", sessionInput.Source)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Bucket != "" || sessionInput.S3Key != "" || sessionInput.SpoolPath != "" {
|
||||||
|
t.Fatalf("local session input has unexpected remote provenance: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRecordsRemoteSessionProvenance(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
remoteSessionPath := filepath.Join(t.TempDir(), "downloaded-session.yml")
|
||||||
|
writeFile(t, remoteSessionPath, "session_id: 2026-05-03\ninputs:\n audio_s3:\n prefix: audio/\n")
|
||||||
|
env.Config.SessionPath = remoteSessionPath
|
||||||
|
env.Config.Session.Inputs.AudioDir = ""
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
env.Config.SessionSource = config.SessionSource{
|
||||||
|
Source: "session_config.s3",
|
||||||
|
LocalPath: remoteSessionPath,
|
||||||
|
S3Bucket: "my-dnd-archive",
|
||||||
|
S3Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml",
|
||||||
|
S3Size: 58,
|
||||||
|
S3ETag: "session-etag",
|
||||||
|
SpoolPath: remoteSessionPath,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||||
|
env.ObjectStore = fake
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||||
|
if sessionInput.Source != "session_config.s3" {
|
||||||
|
t.Fatalf("session source = %q, want session_config.s3", sessionInput.Source)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Bucket != "my-dnd-archive" || sessionInput.S3Key != "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml" {
|
||||||
|
t.Fatalf("remote session input missing bucket/key: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Size != 58 || sessionInput.S3ETag != "session-etag" || sessionInput.SpoolPath != remoteSessionPath {
|
||||||
|
t.Fatalf("remote session input missing metadata: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||||
env, m := setupPrepareEnv(t)
|
env, m := setupPrepareEnv(t)
|
||||||
env.Config.Session.Campaign = "forsaken"
|
env.Config.Session.Campaign = "forsaken"
|
||||||
@@ -186,6 +245,15 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
|||||||
t.Fatalf("expected file %q: %v", p, err)
|
t.Fatalf("expected file %q: %v", p, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if got := result.Metadata["audio_cache_hits"]; got != 0 {
|
||||||
|
t.Fatalf("audio_cache_hits = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := result.Metadata["audio_cache_misses"]; got != 2 {
|
||||||
|
t.Fatalf("audio_cache_misses = %#v, want 2", got)
|
||||||
|
}
|
||||||
|
if got := result.Metadata["audio_s3_downloads"]; got != 2 {
|
||||||
|
t.Fatalf("audio_s3_downloads = %#v, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
audioInputs := 0
|
audioInputs := 0
|
||||||
for _, in := range m.Inputs {
|
for _, in := range m.Inputs {
|
||||||
@@ -199,15 +267,75 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
|||||||
if in.S3Bucket != "my-dnd-archive" {
|
if in.S3Bucket != "my-dnd-archive" {
|
||||||
t.Fatalf("audio input bucket = %q", in.S3Bucket)
|
t.Fatalf("audio input bucket = %q", in.S3Bucket)
|
||||||
}
|
}
|
||||||
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
|
if in.S3Key == "" || in.SpoolPath == "" || in.CachePath == "" || in.Checksum == "" {
|
||||||
t.Fatalf("audio input missing provenance: %#v", in)
|
t.Fatalf("audio input missing provenance: %#v", in)
|
||||||
}
|
}
|
||||||
|
if _, err := os.Stat(in.CachePath); err != nil {
|
||||||
|
t.Fatalf("expected cache path %q: %v", in.CachePath, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if audioInputs != 2 {
|
if audioInputs != 2 {
|
||||||
t.Fatalf("audio input count = %d, want 2", audioInputs)
|
t.Fatalf("audio input count = %d, want 2", audioInputs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageS3AudioUsesCacheOnRerun(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.Inputs.AudioDir = ""
|
||||||
|
env.Config.Session.Inputs.AudioFiles = nil
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||||
|
env.ObjectStore = fake
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("first prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Downloads) != 1 {
|
||||||
|
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.RemoveAll(filepath.Join(m.LocalWorkDir, "audio")); err != nil {
|
||||||
|
t.Fatalf("remove work audio: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(m.LocalSpoolDir); err != nil {
|
||||||
|
t.Fatalf("remove spool audio: %v", err)
|
||||||
|
}
|
||||||
|
fake.DownloadErr = os.ErrPermission
|
||||||
|
|
||||||
|
m2 := manifest.New("2026-05-03", time.Now().UTC())
|
||||||
|
m2.RunID = "20260515T041522Z-a1b2c3d4"
|
||||||
|
m2.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m2.SessionID, m2.RunID)
|
||||||
|
m2.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m2.SessionID, m2.RunID)
|
||||||
|
result, err := (prepareStage{}).Run(context.Background(), env, m2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cached prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Downloads) != 1 {
|
||||||
|
t.Fatalf("downloads = %d, want cached rerun to avoid new download", len(fake.Downloads))
|
||||||
|
}
|
||||||
|
if got := result.Metadata["audio_cache_hits"]; got != 1 {
|
||||||
|
t.Fatalf("audio_cache_hits = %#v, want 1", got)
|
||||||
|
}
|
||||||
|
if got := result.Metadata["audio_s3_downloads"]; got != 0 {
|
||||||
|
t.Fatalf("audio_s3_downloads = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
audioInput := findManifestInput(t, m2.Inputs, "audio")
|
||||||
|
if audioInput.CachePath == "" {
|
||||||
|
t.Fatalf("audio input missing cache path: %#v", audioInput)
|
||||||
|
}
|
||||||
|
if audioInput.SpoolPath != "" {
|
||||||
|
t.Fatalf("audio input spool path = %q, want empty on cache hit", audioInput.SpoolPath)
|
||||||
|
}
|
||||||
|
mustReadFileEquals(t, filepath.Join(m2.LocalWorkDir, "audio", "alice.flac"), "alice")
|
||||||
|
}
|
||||||
|
|
||||||
func TestPrepareStageS3AudioFailures(t *testing.T) {
|
func TestPrepareStageS3AudioFailures(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -382,6 +510,10 @@ func TestPrepareStageHydratesRequiredPreviousArtifactAndRecordsInputs(t *testing
|
|||||||
if _, err := os.Stat(recapPath); err != nil {
|
if _, err := os.Stat(recapPath); err != nil {
|
||||||
t.Fatalf("expected previous artifact: %v", err)
|
t.Fatalf("expected previous artifact: %v", err)
|
||||||
}
|
}
|
||||||
|
nestedRecapPath := filepath.Join(paths.PreviousArtifactsDir, "artifacts", "session_recap.md")
|
||||||
|
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
var hasPreviousManifest, hasPreviousArtifact bool
|
var hasPreviousManifest, hasPreviousArtifact bool
|
||||||
for _, in := range m.Inputs {
|
for _, in := range m.Inputs {
|
||||||
@@ -473,25 +605,51 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
|
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
|
|
||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
|
writeFile(t, campaignPath, `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`)
|
||||||
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
|
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Workspace: config.WorkspaceConfig{Root: workspace},
|
||||||
|
Cache: config.CacheConfig{Root: filepath.Join(t.TempDir(), "cache"), S3Audio: boolPtr(true)},
|
||||||
|
},
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
},
|
||||||
AutocorrectFile: "./autocorrect.yml",
|
},
|
||||||
GlossaryFile: "./glossary.yml",
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -519,3 +677,25 @@ func snapshotInputs(inputs []manifest.InputRecord) map[string]string {
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func findManifestInput(t *testing.T, inputs []manifest.InputRecord, kind string) manifest.InputRecord {
|
||||||
|
t.Helper()
|
||||||
|
for _, input := range inputs {
|
||||||
|
if input.Kind == kind {
|
||||||
|
return input
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("manifest input kind %q not found in %#v", kind, inputs)
|
||||||
|
return manifest.InputRecord{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustReadFileEquals(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %q: %v", path, err)
|
||||||
|
}
|
||||||
|
if string(data) != want {
|
||||||
|
t.Fatalf("%q = %q, want %q", path, string(data), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -233,8 +233,10 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
|
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
|
writeFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
@@ -243,7 +245,9 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
concurrency := 2
|
concurrency := 2
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Pipeline: &config.PipelineConfig{
|
Pipeline: &config.PipelineConfig{
|
||||||
Workspace: config.WorkspaceConfig{Root: workspace},
|
Workspace: config.WorkspaceConfig{Root: workspace},
|
||||||
WhisperX: config.WhisperXConfig{
|
WhisperX: config.WhisperXConfig{
|
||||||
@@ -265,6 +269,11 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
GlossaryFile: "./glossary.yml",
|
GlossaryFile: "./glossary.yml",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{Path: "./speakers.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{Path: "./autocorrect.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{Path: "./glossary.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
env := &Env{
|
env := &Env{
|
||||||
|
|||||||
Reference in New Issue
Block a user