Compare commits
14 Commits
dffb432537
...
v0.12.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 |
276
docs/cli.md
276
docs/cli.md
@@ -6,7 +6,9 @@
|
||||
narratio run --session-id 2026-04-04
|
||||
```
|
||||
|
||||
This command uses default discovery for `pipeline.yml` and `session.yml`; both files must be discoverable unless you pass explicit `--config` and `--session` paths.
|
||||
This command uses default system discovery for `pipeline.yml`, `campaign.yml`, and local `session.yml`. If local session discovery misses and S3 storage is configured, `--session-id` can load remote `session.yml` from the canonical session prefix.
|
||||
|
||||
Default discovery checks system config locations only. Pass `--config`, `--campaign`, and `--session` to use files from the current working directory.
|
||||
|
||||
## Command Overview
|
||||
|
||||
@@ -15,9 +17,15 @@ Implemented commands:
|
||||
- `run`: execute pipeline stages and persist manifest state.
|
||||
- `plan`: validate config, prepare workspace layout, and print stage run/skip decisions.
|
||||
- `resume`: continue from first non-succeeded stage unless forced.
|
||||
- `status`: read and print stage statuses from an existing manifest.
|
||||
- `status`: read an existing manifest or inspect local/remote state for a session.
|
||||
- `run-stage`: execute exactly one stage.
|
||||
- `analyze`: force-rerun the analyze stage.
|
||||
- `restore`: restore durable local session state from the committed remote archive state.
|
||||
- `session validate`: run read-only preflight checks for a session.
|
||||
- `session init`: create local or remote `session.yml`.
|
||||
- `artifacts list`: list effective artifact source IDs.
|
||||
- `locks`: list, add, and remove archive promotion locks.
|
||||
- `clean`: remove local workspace/spool state for one session or all local sessions.
|
||||
|
||||
Unknown commands print usage and exit non-zero.
|
||||
|
||||
@@ -28,6 +36,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
||||
### `run`
|
||||
|
||||
- `--config <path>`: optional explicit `pipeline.yml` path.
|
||||
- `--campaign <path>`: optional explicit `campaign.yml` path.
|
||||
- `--session <path>`: optional explicit `session.yml` path.
|
||||
- `--session-id <value>`: session template variable value.
|
||||
- `--previous-session-id <value>`: previous-session template variable value.
|
||||
@@ -37,6 +46,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
||||
### `plan`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
@@ -45,6 +55,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
||||
### `resume`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
@@ -54,6 +65,7 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
||||
### `run-stage`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
@@ -61,6 +73,17 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||
- positional `<stage>`: required stage name.
|
||||
|
||||
### `analyze`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||
|
||||
`analyze` is force-by-design and does not accept `--force`.
|
||||
|
||||
Valid stage names:
|
||||
|
||||
- `prepare`
|
||||
@@ -76,6 +99,7 @@ Valid stage names:
|
||||
### `restore`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
@@ -83,9 +107,68 @@ Valid stage names:
|
||||
- `--force`: overwrite local conflicting files with remote archive files.
|
||||
- `--include-audio`: include durable archived `audio/**` files in restore scope.
|
||||
|
||||
### `clean`
|
||||
|
||||
- `--session-id <value>`: required for session cleanup unless `--all` is set.
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--previous-session-id <value>`
|
||||
- `--all`: clean all local session work/spool state using pipeline config only.
|
||||
- `--dry-run`: print cleanup targets without deleting.
|
||||
- `--clear-cache`: also remove matching S3 audio cache entries.
|
||||
|
||||
### `status`
|
||||
|
||||
- `--manifest <path>`: required manifest path.
|
||||
- `--manifest <path>`: inspect one manifest file.
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
|
||||
### `session validate`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
|
||||
### `session init`
|
||||
|
||||
- `--config <path>`: required.
|
||||
- `--campaign <path>`: required.
|
||||
- `--session-id <value>`: required.
|
||||
- `--output <path>`: local `session.yml` target; mutually exclusive with `--remote`.
|
||||
- `--remote`: write remote `session.yml` to the canonical session prefix; mutually exclusive with `--output`.
|
||||
- `--previous-session-id <value>`
|
||||
- `--date <value>`
|
||||
- `--title <value>`
|
||||
- `--audio-s3-prefix <prefix>`: defaults to `audio/` when neither audio flag is provided.
|
||||
- `--audio-dir <path>`: local audio directory; mutually exclusive with `--audio-s3-prefix`.
|
||||
- `--force`: overwrite existing local or remote target.
|
||||
|
||||
### `artifacts list`
|
||||
|
||||
- `--config <path>`
|
||||
- `--campaign <path>`
|
||||
- `--session <path>`
|
||||
- `--session-id <value>`
|
||||
- `--previous-session-id <value>`
|
||||
- `--remote`: check remote availability for configured archive promotion destinations.
|
||||
|
||||
### `locks`
|
||||
|
||||
- `--session-id <value>`: required for list, add, and remove.
|
||||
- `--config <path>`: optional explicit `pipeline.yml` path.
|
||||
- `--campaign <path>`: optional explicit `campaign.yml` path.
|
||||
- `--session <path>`: optional explicit `session.yml` path.
|
||||
- `--previous-session-id <value>`: optional session template value.
|
||||
- `add <source>`: add a remote lock for one artifact or transcript source.
|
||||
- `add --reason <text>`: record an optional remote lock reason.
|
||||
- `add --force`: update the reason for an existing remote lock.
|
||||
- `remove <source>`: remove one remote lock.
|
||||
|
||||
## Command Reference
|
||||
|
||||
@@ -97,14 +180,15 @@ Purpose:
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio run [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||
narratio run [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||
```
|
||||
|
||||
Success output:
|
||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||
|
||||
Common failure cases:
|
||||
- missing default config/session paths when flags omitted.
|
||||
- missing system default config/campaign/session paths when flags omitted.
|
||||
- missing local session plus missing/unavailable remote `session.yml`.
|
||||
- invalid template/rendered session mismatch.
|
||||
- unknown/invalid `--artifacts` value.
|
||||
- `--artifacts` with unknown configured artifact key.
|
||||
@@ -117,7 +201,7 @@ Purpose:
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio plan [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
|
||||
narratio plan [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
|
||||
```
|
||||
|
||||
Success output includes:
|
||||
@@ -126,7 +210,8 @@ Success output includes:
|
||||
- `totals: run=<n> skip=<n>`
|
||||
|
||||
Common failure cases:
|
||||
- same config/session discovery and validation failures as `run`.
|
||||
- same config/campaign/session discovery and validation failures as `run`.
|
||||
- remote session fallback failures when local session discovery misses.
|
||||
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
|
||||
|
||||
### `resume`
|
||||
@@ -137,7 +222,7 @@ Purpose:
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio resume [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||
narratio resume [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||
```
|
||||
|
||||
Success output:
|
||||
@@ -152,22 +237,111 @@ Common failure cases:
|
||||
### `status`
|
||||
|
||||
Purpose:
|
||||
- Inspect one manifest file without executing stages.
|
||||
- Inspect one manifest file, or inspect configured local/remote state for a session.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio status --manifest <manifest.json>
|
||||
narratio status [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
|
||||
```
|
||||
|
||||
Success output includes:
|
||||
Manifest output includes:
|
||||
- `session_id: <id>`
|
||||
- `updated_at: <timestamp>`
|
||||
- `stages:` entries (`- <stage>: <status>`)
|
||||
|
||||
Session output includes:
|
||||
- session ID, campaign, workspace, session config source.
|
||||
- local manifest state when present.
|
||||
- remote current archive state when storage is configured.
|
||||
- catalog-based remote output availability for expected transcript and artifact sources.
|
||||
- effective archive locks and conservative next actions.
|
||||
|
||||
Common failure cases:
|
||||
- missing `--manifest`.
|
||||
- missing `--manifest` when no config/session flags are provided.
|
||||
- unreadable or invalid manifest path.
|
||||
- invalid config or remote session fallback failure in session mode.
|
||||
|
||||
### `session validate`
|
||||
|
||||
Purpose:
|
||||
- Run read-only preflight checks for a session.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio session validate [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
|
||||
```
|
||||
|
||||
Checks include:
|
||||
- effective config and session source.
|
||||
- stable input files.
|
||||
- local or remote audio availability.
|
||||
- previous-session requirements.
|
||||
- archive promotions and effective locks.
|
||||
|
||||
Warnings do not fail the command. Any `ERROR` finding exits non-zero.
|
||||
|
||||
### `session init`
|
||||
|
||||
Purpose:
|
||||
- Create a strict-decoded session skeleton locally or in object storage.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --output ./session.yml
|
||||
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --remote
|
||||
```
|
||||
|
||||
Behavior:
|
||||
- exactly one of `--output` or `--remote` is required.
|
||||
- remote writes target `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
||||
- existing local or remote targets fail unless `--force` is passed.
|
||||
- remote writes use existence checks, not compare-and-swap.
|
||||
|
||||
### `artifacts list`
|
||||
|
||||
Purpose:
|
||||
- List built-in, configured, previous-session, promoted, and locked artifact sources.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio artifacts list [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--remote]
|
||||
```
|
||||
|
||||
`--remote` checks promoted top-level object availability through the storage adapter. Remote markers appear only in the `Promoted` section, which reports each configured archive promotion destination and includes `dest=<path>` when that destination differs from the source's canonical path.
|
||||
|
||||
### `locks`
|
||||
|
||||
Purpose:
|
||||
- Inspect and mutate source-based archive promotion locks for one session.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio locks --session-id <id>
|
||||
narratio locks add --session-id <id> [--reason <text>] [--force] <source>
|
||||
narratio locks remove --session-id <id> <source>
|
||||
```
|
||||
|
||||
Behavior:
|
||||
- `--session-id` is required for list, add, and remove.
|
||||
- optional `--config`, `--campaign`, and `--session` override default config discovery.
|
||||
- list mode prints effective locks from static `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`.
|
||||
- `locks add` writes only the remote lock store and fails if the source is already locked by pipeline config.
|
||||
- `locks remove` removes only remote locks and cannot remove static pipeline locks.
|
||||
- `locks add --force` is required to update an existing remote lock reason.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
narratio locks --session-id 2026-04-04
|
||||
narratio locks add --session-id 2026-04-04 --reason "manual transcript review" narratio.transcript.trimmed
|
||||
narratio locks remove --session-id 2026-04-04 narratio.transcript.trimmed
|
||||
```
|
||||
|
||||
### `run-stage`
|
||||
|
||||
@@ -177,7 +351,7 @@ Purpose:
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio run-stage [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
||||
narratio run-stage [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
||||
```
|
||||
|
||||
Success output:
|
||||
@@ -193,6 +367,26 @@ Common failure cases:
|
||||
- unknown stage name.
|
||||
- using `--artifacts` with any non-`analyze` stage.
|
||||
|
||||
### `analyze`
|
||||
|
||||
Purpose:
|
||||
- Force-rerun the analyze stage.
|
||||
- Provide a shorter equivalent for `narratio run-stage --force analyze`.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio analyze [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
|
||||
```
|
||||
|
||||
Success output:
|
||||
- `narratio analyze: executed=<n> skipped=<n> force=true; manifest=<path>`
|
||||
|
||||
Common failure cases:
|
||||
- positional arguments.
|
||||
- `--force`, because force is implicit.
|
||||
- unknown configured artifact keys.
|
||||
|
||||
### `restore`
|
||||
|
||||
Purpose:
|
||||
@@ -201,7 +395,7 @@ Purpose:
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio restore [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
||||
narratio restore [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
||||
```
|
||||
|
||||
Success output (dry-run):
|
||||
@@ -226,6 +420,40 @@ Common failure cases:
|
||||
- local conflicts without `--force`.
|
||||
- session lock conflict.
|
||||
|
||||
When `--include-audio` is set, S3 audio files are restored through the shared audio cache. Cache hits avoid re-downloading large audio objects.
|
||||
|
||||
### `clean`
|
||||
|
||||
Purpose:
|
||||
- Remove local Narratio work/spool state for testing, reruns, or recovery from corrupted local files.
|
||||
- Preserve durable S3 audio cache state unless `--clear-cache` is passed.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
narratio clean --session-id <id> [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--clear-cache]
|
||||
narratio clean --all [--config <pipeline.yml>] [--dry-run] [--clear-cache]
|
||||
```
|
||||
|
||||
Session cleanup deletes:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
- `{spool.root}/{campaign}/{session_id}`
|
||||
|
||||
All-session cleanup deletes:
|
||||
- `{workspace.root}/work`
|
||||
- the contents of `{spool.root}`, while preserving the spool root directory itself.
|
||||
|
||||
Cache behavior:
|
||||
- cache is preserved by default.
|
||||
- `--clear-cache` in session mode removes cached S3 audio files for the resolved session.
|
||||
- `--all --clear-cache` removes the configured Narratio S3 audio cache namespace for the configured bucket/root prefix.
|
||||
- `--clear-cache` does not delete arbitrary files under `pipeline.cache.root`.
|
||||
|
||||
Common failure cases:
|
||||
- missing `--session-id` when `--all` is not set.
|
||||
- combining `--all` with `--campaign`, `--session`, `--session-id`, or `--previous-session-id`.
|
||||
- unsafe cleanup target, such as a symlink, a non-directory session target, a configured root directory, or a path outside the configured root.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
Default-discovery run:
|
||||
@@ -246,10 +474,10 @@ Resume with selected analyze artifacts:
|
||||
narratio resume --session-id 2026-04-04 --artifacts player_handout
|
||||
```
|
||||
|
||||
Run only analyze stage with selected artifacts:
|
||||
Force-rerun analyze with selected artifacts:
|
||||
|
||||
```bash
|
||||
narratio run-stage --session-id 2026-04-04 --artifacts player_handout analyze
|
||||
narratio analyze --session-id 2026-04-04 --artifacts player_handout
|
||||
```
|
||||
|
||||
Preview restore actions without writes:
|
||||
@@ -262,7 +490,7 @@ Restore and then force analyze:
|
||||
|
||||
```bash
|
||||
narratio restore --session-id 2026-04-04
|
||||
narratio run-stage --session-id 2026-04-04 --force analyze
|
||||
narratio analyze --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Rehydrate canonical previous-session inputs after artifact-input changes:
|
||||
@@ -271,6 +499,20 @@ Rehydrate canonical previous-session inputs after artifact-input changes:
|
||||
narratio run-stage --session-id 2026-04-04 --force prepare
|
||||
```
|
||||
|
||||
Reset local state before testing restore:
|
||||
|
||||
```bash
|
||||
narratio clean --session-id 2026-04-04 --dry-run
|
||||
narratio clean --session-id 2026-04-04
|
||||
narratio restore --session-id 2026-04-04 --include-audio
|
||||
```
|
||||
|
||||
Clean all local sessions while keeping cached S3 audio:
|
||||
|
||||
```bash
|
||||
narratio clean --all
|
||||
```
|
||||
|
||||
## Diagnostic / Recovery Commands
|
||||
|
||||
Inspect stage status:
|
||||
@@ -280,7 +522,7 @@ narratio status --manifest <manifest.json>
|
||||
```
|
||||
|
||||
Get manifest path from previous output:
|
||||
- `run`, `resume`, and `run-stage` print `manifest=<path>` on success.
|
||||
- `run`, `resume`, `run-stage`, and `analyze` print `manifest=<path>` on success.
|
||||
|
||||
## `--artifacts` and `--force`
|
||||
|
||||
|
||||
190
docs/config.md
190
docs/config.md
@@ -2,12 +2,13 @@
|
||||
|
||||
## 1. Overview
|
||||
|
||||
Narratio loads two YAML files:
|
||||
Narratio loads three YAML files:
|
||||
|
||||
- `pipeline.yml`: pipeline-level runtime configuration.
|
||||
- `session.yml`: per-session metadata and input selection.
|
||||
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
|
||||
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
|
||||
|
||||
These commands load and validate both files before running:
|
||||
These commands load and validate all three files before running:
|
||||
|
||||
- `narratio run`
|
||||
- `narratio plan`
|
||||
@@ -19,12 +20,23 @@ Behavior:
|
||||
|
||||
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
||||
- session templates render before session YAML decode.
|
||||
- remote `session.yml` uses the same strict decode and template behavior as local `session.yml`.
|
||||
- defaults are applied for optional pipeline fields.
|
||||
- campaign-level stable input paths fill missing session input paths.
|
||||
- session-level stable input paths override campaign-level input paths.
|
||||
- validation enforces required fields, value formats, and cross-field constraints.
|
||||
|
||||
## 2. Config file discovery
|
||||
|
||||
Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
||||
These commands use the same config discovery behavior:
|
||||
|
||||
- `narratio run`
|
||||
- `narratio plan`
|
||||
- `narratio resume`
|
||||
- `narratio run-stage`
|
||||
- `narratio restore`
|
||||
|
||||
Pipeline config lookup:
|
||||
|
||||
- if `--config <path>` is provided, that path is used.
|
||||
- if omitted, Narratio searches in order:
|
||||
@@ -32,18 +44,29 @@ Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
||||
2. `/etc/narratio/pipeline.yml`
|
||||
- first existing file wins.
|
||||
|
||||
## 3. Session file discovery and templating
|
||||
Campaign config lookup:
|
||||
|
||||
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
||||
|
||||
- if `--session <path>` is provided, that path is used.
|
||||
- if `--campaign <path>` is provided, that path is used.
|
||||
- if omitted, Narratio searches in order:
|
||||
1. `./session.yml`
|
||||
2. `/usr/local/etc/narratio/session.yml`
|
||||
3. `/etc/narratio/session.yml`
|
||||
1. `/usr/local/etc/narratio/campaign.yml`
|
||||
2. `/etc/narratio/campaign.yml`
|
||||
- first existing file wins.
|
||||
|
||||
Template behavior:
|
||||
Session config lookup:
|
||||
|
||||
- if `--session <path>` is provided, that path is used.
|
||||
- if `--session` is omitted, Narratio searches locally in order:
|
||||
1. `/usr/local/etc/narratio/session.yml`
|
||||
2. `/etc/narratio/session.yml`
|
||||
- first existing local file wins.
|
||||
- if no local session file is found, `--session-id <value>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
|
||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||
- local discovery always runs before remote fallback.
|
||||
- local files in the current working directory are used only when passed explicitly, for example `--config ./pipeline.yml --campaign ./campaign.yml --session ./session.yml`.
|
||||
|
||||
## 3. Session templating
|
||||
|
||||
Template behavior for local and remote `session.yml`:
|
||||
|
||||
- supported placeholders:
|
||||
- `{{session_id}}`
|
||||
@@ -56,7 +79,9 @@ Template behavior:
|
||||
- if rendered `session_id` mismatches `--session-id`, load fails.
|
||||
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
|
||||
|
||||
## 4. Minimal pipeline config
|
||||
## 4. Minimal config set
|
||||
|
||||
### `pipeline.yml`
|
||||
|
||||
```yaml
|
||||
whisperx:
|
||||
@@ -69,22 +94,40 @@ Why this is sufficient:
|
||||
- `workspace.root` defaults to `/var/lib/narratio`.
|
||||
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
|
||||
|
||||
## 5. Minimal session template
|
||||
### `campaign.yml`
|
||||
|
||||
```yaml
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
```
|
||||
|
||||
Why this is sufficient:
|
||||
|
||||
- `campaign` supplies the stable campaign identity.
|
||||
- stable input files are required and resolve relative to `campaign.yml` when copied during `prepare`.
|
||||
|
||||
### `session.yml`
|
||||
|
||||
```yaml
|
||||
session_id: "{{ session_id }}"
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./examples/speakers.yml
|
||||
autocorrect_file: ./examples/autocorrect.yml
|
||||
glossary_file: ./examples/glossary.yml
|
||||
```
|
||||
|
||||
Usage:
|
||||
Why this is sufficient:
|
||||
|
||||
- `session_id` is required and can be rendered from `--session-id`.
|
||||
- `campaign` can be omitted because it is supplied by `campaign.yml`.
|
||||
- stable input paths can be omitted because `campaign.yml` supplies defaults.
|
||||
- local `audio_dir` resolves relative to `session.yml`.
|
||||
|
||||
Minimal local-file usage:
|
||||
|
||||
```bash
|
||||
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03
|
||||
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03
|
||||
```
|
||||
|
||||
Previous-session-enabled variant:
|
||||
@@ -92,19 +135,17 @@ Previous-session-enabled variant:
|
||||
```yaml
|
||||
session_id: "{{ session_id }}"
|
||||
previous_session_id: "{{ previous_session_id }}"
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./examples/speakers.yml
|
||||
autocorrect_file: ./examples/autocorrect.yml
|
||||
glossary_file: ./examples/glossary.yml
|
||||
```
|
||||
|
||||
```bash
|
||||
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||
```
|
||||
|
||||
## 6. Production-oriented config
|
||||
## 5. Production-oriented config set
|
||||
|
||||
### `pipeline.yml`
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
@@ -124,6 +165,10 @@ spool:
|
||||
root: /var/spool/narratio
|
||||
delete_audio_after_archive: true
|
||||
|
||||
cache:
|
||||
root: /var/cache/narratio
|
||||
s3_audio: true
|
||||
|
||||
archive:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
@@ -134,6 +179,9 @@ archive:
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: Final recap was manually edited.
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
@@ -153,14 +201,53 @@ scriptorium:
|
||||
required: false
|
||||
```
|
||||
|
||||
### `campaign.yml`
|
||||
|
||||
```yaml
|
||||
campaign: forsaken
|
||||
inputs:
|
||||
speakers_file: /srv/narratio/campaigns/forsaken/speakers.yml
|
||||
autocorrect_file: /srv/narratio/campaigns/forsaken/autocorrect.yml
|
||||
glossary_file: /srv/narratio/campaigns/forsaken/glossary.yml
|
||||
```
|
||||
|
||||
### Local `session.yml`
|
||||
|
||||
```yaml
|
||||
session_id: "{{ session_id }}"
|
||||
previous_session_id: "{{ previous_session_id }}"
|
||||
date: 2026-05-03
|
||||
title: The Black Cabin
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
```
|
||||
|
||||
### S3-first session config
|
||||
|
||||
For S3-first operation, upload the same `session.yml` content to:
|
||||
|
||||
```text
|
||||
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml
|
||||
```
|
||||
|
||||
Then run with explicit or discovered pipeline/campaign config and no `--session`:
|
||||
|
||||
```bash
|
||||
narratio run --config /usr/local/etc/narratio/pipeline.yml --campaign /usr/local/etc/narratio/campaign.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||
```
|
||||
|
||||
Operational notes:
|
||||
|
||||
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
||||
- `source` is required; `dest` is optional and derived when omitted.
|
||||
- `archive.locks` skips top-level promotion overwrites for static locked sources while preserving run-local uploads.
|
||||
- operator-created mutable locks are stored at `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml` and are merged with static locks.
|
||||
- Narratio does not auto-promote all generated analyze artifacts.
|
||||
- `restore` reads the same config/session inputs and restore scope is bounded by committed archive current state.
|
||||
- `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
|
||||
- `clean` removes workspace/spool state by default and preserves `pipeline.cache.root` unless `--clear-cache` is passed.
|
||||
|
||||
## 7. Full pipeline reference
|
||||
## 6. Full pipeline reference
|
||||
|
||||
| Path | Type | Required | Default |
|
||||
| --- | --- | --- | --- |
|
||||
@@ -179,12 +266,17 @@ Operational notes:
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.archive.enabled` | bool | No | `true` |
|
||||
| `pipeline.archive.upload_run` | bool | No | `true` |
|
||||
| `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule |
|
||||
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
||||
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
||||
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
||||
| `pipeline.archive.locks[]` | list | No | empty |
|
||||
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
|
||||
| `pipeline.archive.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
||||
@@ -285,6 +377,8 @@ Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
||||
- `narratio.bounds.session`
|
||||
- `narratio.artifact.<configured_artifact_key>`
|
||||
|
||||
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
|
||||
|
||||
Archive promotion destination rules:
|
||||
|
||||
- `dest` must be a clean relative path (not absolute, no traversal).
|
||||
@@ -294,10 +388,39 @@ Archive promotion destination rules:
|
||||
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
||||
- derivation failure is a config validation error.
|
||||
|
||||
Archive lock rules:
|
||||
|
||||
- locks are source-based and do not accept `dest`.
|
||||
- duplicate lock sources are rejected.
|
||||
- static `pipeline.archive.locks` win over remote mutable locks for the same source.
|
||||
- locked promotions are recorded as intentional skips in archive metadata.
|
||||
- locked required promotions do not fail archive by default.
|
||||
- ordinary `--force` reruns do not override locks.
|
||||
|
||||
Remote mutable lock store:
|
||||
|
||||
- path: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml`.
|
||||
- strict YAML shape: top-level `locks`, each with `source` and optional `reason`.
|
||||
- `narratio locks add` and `narratio locks remove` mutate only the remote lock store.
|
||||
- writes use existence checks plus `--force` for updates; they are not compare-and-swap atomic.
|
||||
|
||||
Restore-related implications:
|
||||
|
||||
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
||||
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
|
||||
- S3 audio downloads use `pipeline.spool.root` for active downloads and `pipeline.cache.root` for reusable cached audio when `pipeline.cache.s3_audio` is true.
|
||||
- `pipeline.cache.root` is durable local cache state. It is not workspace state and is preserved by default by `narratio clean`.
|
||||
|
||||
## 7. Full campaign reference
|
||||
|
||||
| Path | Type | Required | Default |
|
||||
| --- | --- | --- | --- |
|
||||
| `campaign.campaign` | string | Yes | none |
|
||||
| `campaign.inputs.speakers_file` | string | Yes | none |
|
||||
| `campaign.inputs.autocorrect_file` | string | Yes | none |
|
||||
| `campaign.inputs.glossary_file` | string | Yes | none |
|
||||
|
||||
Campaign input paths may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
|
||||
|
||||
## 8. Full session reference
|
||||
|
||||
@@ -305,15 +428,17 @@ Restore-related implications:
|
||||
| --- | --- | --- | --- |
|
||||
| `session.session_id` | string | Yes | none |
|
||||
| `session.previous_session_id` | string | No | empty |
|
||||
| `session.campaign` | string | Yes | none |
|
||||
| `session.campaign` | string | No | `campaign.campaign` |
|
||||
| `session.date` | string | No | empty |
|
||||
| `session.title` | string | No | empty |
|
||||
| `session.inputs.audio_dir` | string | Conditional | empty |
|
||||
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
||||
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
||||
| `session.inputs.speakers_file` | string | Yes | none |
|
||||
| `session.inputs.autocorrect_file` | string | Yes | none |
|
||||
| `session.inputs.glossary_file` | string | Yes | none |
|
||||
| `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
|
||||
| `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
|
||||
| `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
|
||||
|
||||
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
|
||||
|
||||
Audio-source rule:
|
||||
|
||||
@@ -354,6 +479,7 @@ Maintained examples:
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/campaign.yml`
|
||||
- `examples/session.template.yml`
|
||||
- `examples/session.local-audio.yml`
|
||||
- `examples/session.s3-audio.yml`
|
||||
|
||||
@@ -69,6 +69,8 @@ For design principles and invariants, see [docs/architecture.md](./architecture.
|
||||
2. Add or update command tests (`TestExecute` and command-specific tests).
|
||||
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
||||
|
||||
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
|
||||
|
||||
### Add or modify stages/adapters
|
||||
|
||||
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||
|
||||
@@ -47,6 +47,7 @@ Current execution usage:
|
||||
|
||||
Default construction in app runner:
|
||||
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
|
||||
- Object-store construction goes through app command orchestration so configured filesystem secrets are loaded before the storage adapter is initialized.
|
||||
- Callers can inject test/fake implementations through `app.RunOptions.Env`.
|
||||
|
||||
## State and manifest behavior
|
||||
|
||||
@@ -16,6 +16,7 @@ Outputs:
|
||||
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
||||
- runtime catalog entries for built-ins and configured artifacts;
|
||||
- requirement sets for canonical previous-session inputs.
|
||||
- canonical S3 session, run, current, session config, session locks, audio, and promoted artifact keys.
|
||||
|
||||
## Boundaries
|
||||
Owns:
|
||||
@@ -43,6 +44,15 @@ Does not own:
|
||||
- configured artifact: `narratio.artifact.<artifact_key>`
|
||||
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
## S3 key helpers
|
||||
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||
- session config: `{session_prefix}/session.yml`
|
||||
- session lock store: `{session_prefix}/locks.yml`
|
||||
- run prefix: `{session_prefix}/runs/{run_id}/`
|
||||
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
|
||||
- current manifest: `{session_prefix}/current/manifest.json`
|
||||
- current run pointer: `{session_prefix}/current/run_id.txt`
|
||||
|
||||
## Runtime catalog model
|
||||
Catalog entries track:
|
||||
- `planned`: source is registered for this run;
|
||||
@@ -61,6 +71,7 @@ Previous-session canonical provenance values include:
|
||||
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
||||
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
||||
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
||||
- Archive-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
|
||||
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
||||
|
||||
## Previous-session requirement scanning
|
||||
|
||||
@@ -34,6 +34,8 @@ Does not own:
|
||||
- `pipeline.workspace.root` (local restore target root).
|
||||
- `pipeline.storage.*` (remote backend + archive identity derivation).
|
||||
- `pipeline.storage.s3.*` identity components used by archive prefix helpers.
|
||||
- `pipeline.spool.root` for active audio downloads.
|
||||
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
|
||||
- `session.session_id`
|
||||
- `session.campaign`
|
||||
|
||||
@@ -48,6 +50,7 @@ Does not own:
|
||||
- `current/run_id.txt` must exist and be non-empty.
|
||||
- `current/manifest.json` must decode and match requested session/campaign.
|
||||
- Non-dry-run writes restore files to canonical session paths.
|
||||
- With `--include-audio`, restore uses the shared S3 audio cache for `audio/**` objects. Cache hits avoid object downloads; cache misses download through spool, install the work file, and populate cache.
|
||||
- Manifest install behavior:
|
||||
- validated before replacement.
|
||||
- installed last among download actions.
|
||||
@@ -72,6 +75,7 @@ Restore path scope:
|
||||
## Skip and resume behavior
|
||||
- Restore does not participate in stage skip/resume decisions.
|
||||
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
|
||||
- Audio cache is outside the workspace and is reused across restore and prepare invocations.
|
||||
- Dry-run is read-only and returns plan output only.
|
||||
|
||||
## Failure behavior
|
||||
|
||||
@@ -8,6 +8,7 @@ Inputs:
|
||||
- session manifest and prerequisite stage records
|
||||
- run root contents under `runs/{run_id}/`
|
||||
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
||||
- effective source-based promotion locks from static config and remote session lock store
|
||||
- session-level `previous/**` cache files when present
|
||||
|
||||
Outputs:
|
||||
@@ -23,6 +24,7 @@ Owns:
|
||||
- Prerequisite stage success enforcement
|
||||
- Run file collection and upload (excluding `audio/`)
|
||||
- Promotion rule resolution and upload
|
||||
- Promotion lock enforcement
|
||||
- Session previous-cache file collection/upload
|
||||
- Commit pointer publish order
|
||||
|
||||
@@ -34,6 +36,8 @@ Does not own:
|
||||
- `pipeline.archive.enabled`
|
||||
- `pipeline.archive.upload_run`
|
||||
- `pipeline.archive.promote_artifacts`
|
||||
- `pipeline.archive.locks`
|
||||
- `{session_prefix}/locks.yml` loaded by app orchestration before archive execution
|
||||
- `pipeline.storage.s3.bucket`
|
||||
- `pipeline.storage.s3.root_prefix`
|
||||
- `pipeline.workspace.root`
|
||||
@@ -47,9 +51,12 @@ Does not own:
|
||||
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
||||
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
||||
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
||||
- Skips top-level promotion uploads for effective locked sources; run-local uploads still publish.
|
||||
- Effective locks are the union of `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
|
||||
- Writes metadata including:
|
||||
- upload counts/paths
|
||||
- `previous_files_uploaded` and `previous_uploaded_paths`
|
||||
- `locked_promotion_count` and `locked_promotions`
|
||||
- `current_manifest_key`
|
||||
- `current_run_id_key`
|
||||
- `current_pointer_written`
|
||||
@@ -60,7 +67,8 @@ Does not own:
|
||||
- Runner-level skip also applies for previously succeeded stage unless forced.
|
||||
|
||||
## Failure Behavior
|
||||
- Fails on missing prerequisite success, missing object store when required, missing run root, missing required promotion source, upload failures, or pointer write failures.
|
||||
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
|
||||
- Locked required promotions are intentional skips and do not fail archive.
|
||||
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
||||
|
||||
## Tests to Inspect Before Changing
|
||||
@@ -70,5 +78,7 @@ Does not own:
|
||||
## Architectural Invariants
|
||||
- Run upload excludes `audio/` subtree.
|
||||
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
|
||||
- Ordinary `--force` does not override archive locks.
|
||||
- Malformed or unreadable remote lock store fails archive-capable execution before promotion.
|
||||
- `current/manifest.json` uploads before `current/run_id.txt`.
|
||||
- `current/run_id.txt` is the remote publish commit marker.
|
||||
|
||||
@@ -10,8 +10,9 @@ Prepare owns:
|
||||
|
||||
## Inputs and outputs
|
||||
Inputs:
|
||||
- resolved config/session (`pipeline.yml`, `session.yml`);
|
||||
- session-local input files (`speakers`, `autocorrect`, `glossary`);
|
||||
- resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
|
||||
- remote session provenance when `session.yml` was loaded from S3;
|
||||
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
|
||||
- audio source:
|
||||
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
||||
- S3: `session.inputs.audio_s3.prefix`;
|
||||
@@ -19,6 +20,7 @@ Inputs:
|
||||
- remote previous-session current archive state when previous hydration is required.
|
||||
|
||||
Outputs:
|
||||
- `inputs/campaign.yml`;
|
||||
- `inputs/session.yml`;
|
||||
- `inputs/pipeline.resolved.yml`;
|
||||
- `inputs/speakers.yml`;
|
||||
@@ -53,11 +55,17 @@ Does not own:
|
||||
- `session.inputs.audio_s3.prefix`
|
||||
- `pipeline.workspace.root`
|
||||
- `pipeline.spool.root`
|
||||
- `pipeline.cache.root`
|
||||
- `pipeline.cache.s3_audio`
|
||||
- `pipeline.storage.s3.bucket`
|
||||
- `pipeline.storage.s3.root_prefix`
|
||||
- `pipeline.scriptorium.artifacts.<name>.enabled`
|
||||
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
|
||||
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
|
||||
- `campaign.campaign`
|
||||
- `campaign.inputs.speakers_file`
|
||||
- `campaign.inputs.autocorrect_file`
|
||||
- `campaign.inputs.glossary_file`
|
||||
|
||||
## External adapters used
|
||||
- `storage.ObjectStore` for:
|
||||
@@ -67,15 +75,21 @@ Does not own:
|
||||
## State and manifest behavior
|
||||
- Ensures workspace layout exists.
|
||||
- Materializes canonical input files and audio files.
|
||||
- For S3 audio, uses run-scoped spool for active downloads and durable cache for reusable audio files; cache hits copy directly to work audio without downloading the object again.
|
||||
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
|
||||
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
|
||||
- Resolves session-provided stable input overrides relative to `session.yml`.
|
||||
- Scans enabled configured artifact inputs for canonical sources:
|
||||
- `narratio.previous_session.artifact.<artifact_key>`
|
||||
- If one or more canonical previous-session requirements exist:
|
||||
- clears managed `previous/` state;
|
||||
- hydrates required/optional previous artifacts from the configured previous session’s committed archive current state;
|
||||
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
||||
- stores archive-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
|
||||
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
|
||||
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
||||
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
||||
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
|
||||
|
||||
## Required and optional previous-session behavior
|
||||
- `previous_session_id` unset:
|
||||
|
||||
@@ -6,7 +6,8 @@ Document Narratio's remote storage backend contracts and implementations under `
|
||||
## Inputs and outputs
|
||||
Inputs:
|
||||
- Resolved storage config (`pipeline.storage.*`).
|
||||
- Bucket-relative object keys and local file paths from stage/app orchestration.
|
||||
- Already-loaded environment variables for configured S3 credentials.
|
||||
- Bucket-relative object keys and local file paths from app/stage orchestration.
|
||||
|
||||
Outputs:
|
||||
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
|
||||
@@ -22,6 +23,7 @@ Does not own:
|
||||
- Session/run prefix semantics.
|
||||
- Archive commit order semantics.
|
||||
- Manifest updates.
|
||||
- Filesystem secret loading from `pipeline.secrets.env_dir`.
|
||||
|
||||
## Config fields used
|
||||
- `pipeline.storage.backend`
|
||||
@@ -55,8 +57,10 @@ Implementations:
|
||||
## Failure behavior
|
||||
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
|
||||
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
|
||||
- App command orchestration loads configured filesystem secrets before calling the object-store factory.
|
||||
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
|
||||
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
|
||||
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
|
||||
|
||||
## Tests to inspect before changing
|
||||
- `internal/adapters/storage/factory_test.go`
|
||||
|
||||
@@ -32,6 +32,8 @@ Does not own:
|
||||
- `pipeline.workspace.cleanup_after_archive`
|
||||
- `pipeline.spool.root`
|
||||
- `pipeline.spool.delete_audio_after_archive`
|
||||
- `pipeline.cache.root`
|
||||
- `pipeline.cache.s3_audio`
|
||||
- `session.campaign`
|
||||
- `session.session_id`
|
||||
|
||||
@@ -43,7 +45,11 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
||||
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
|
||||
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
|
||||
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
|
||||
- For S3 audio sessions, `prepare` records spool/work paths and S3 provenance in `manifest.Inputs`.
|
||||
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
|
||||
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
|
||||
- `narratio clean --session-id <id>` removes the session work root and session spool root.
|
||||
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
|
||||
- `narratio clean --clear-cache` is the explicit opt-in for deleting matching S3 audio cache entries.
|
||||
|
||||
## Skip and Resume Behavior
|
||||
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
|
||||
@@ -67,4 +73,5 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
||||
- Run roots are always nested: `runs/{run_id}` under the session root.
|
||||
- Run-local output promotion must end in canonical session paths.
|
||||
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
||||
- Cleanup only targets run-scoped directories and must never delete configured root directories.
|
||||
- Automatic post-archive cleanup only targets run-scoped directories and must never delete configured root directories.
|
||||
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.
|
||||
|
||||
@@ -6,21 +6,37 @@ For field-level configuration, see [docs/config.md](./config.md). For full comma
|
||||
|
||||
## Normal workflow (S3-first path)
|
||||
|
||||
1. Upload session `.flac` files to object storage under the configured session audio prefix.
|
||||
2. Run Narratio:
|
||||
1. Create or upload `session.yml`, or pass a local `session.yml` explicitly.
|
||||
2. Upload session `.flac` files to object storage under the configured session audio prefix.
|
||||
3. Run Narratio:
|
||||
|
||||
```bash
|
||||
narratio run --session-id 2026-04-04
|
||||
```
|
||||
|
||||
3. Read success output:
|
||||
4. Read success output:
|
||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||
- use `manifest=<path>` with `status` for inspection.
|
||||
|
||||
Notes:
|
||||
- default config/session discovery applies unless `--config` and `--session` are passed.
|
||||
- default config/campaign/session discovery checks system config locations unless `--config`, `--campaign`, and `--session` are passed.
|
||||
- when local `session.yml` discovery misses, `--session-id` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
||||
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
||||
|
||||
Initialize a remote session skeleton:
|
||||
|
||||
```bash
|
||||
narratio session init --config /etc/narratio/pipeline.yml --campaign /etc/narratio/campaign.yml --session-id 2026-04-04 --remote
|
||||
```
|
||||
|
||||
Remote init writes `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`. It fails if the object already exists unless `--force` is passed.
|
||||
|
||||
Validate before running:
|
||||
|
||||
```bash
|
||||
narratio session validate --session-id 2026-04-04
|
||||
```
|
||||
|
||||
## Restore workflow
|
||||
|
||||
Use restore when local durable session state is missing or stale and archive current state is authoritative.
|
||||
@@ -40,7 +56,7 @@ narratio restore --session-id 2026-04-04
|
||||
Post-restore analyze rerun pattern:
|
||||
|
||||
```bash
|
||||
narratio run-stage --session-id 2026-04-04 --force analyze
|
||||
narratio analyze --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Restore source-of-truth:
|
||||
@@ -52,6 +68,16 @@ Restore default scope:
|
||||
- includes `audio/**` only with `--include-audio`
|
||||
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
||||
|
||||
Reset local state before restore testing:
|
||||
|
||||
```bash
|
||||
narratio clean --session-id 2026-04-04 --dry-run
|
||||
narratio clean --session-id 2026-04-04
|
||||
narratio restore --session-id 2026-04-04 --include-audio
|
||||
```
|
||||
|
||||
`clean` removes the local session work directory and session spool directory. It preserves the durable S3 audio cache by default, so repeated restore or forced prepare tests do not re-download large audio files.
|
||||
|
||||
## Local filesystem layout and state artifacts
|
||||
|
||||
Session root:
|
||||
@@ -61,6 +87,7 @@ Primary state:
|
||||
- `manifest.json`: session-level stage state.
|
||||
- `runs/{run_id}/manifest.json`: invocation-level state.
|
||||
- `.lock`: session lock while a modifying command is active.
|
||||
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
|
||||
|
||||
Canonical session directories:
|
||||
- `inputs/`
|
||||
@@ -97,8 +124,9 @@ Configured artifact source reuse:
|
||||
- reused configured artifact provenance is `filesystem.disabled_artifact_output`.
|
||||
|
||||
`--artifacts` behavior:
|
||||
- accepted on `run`, `resume`, and `run-stage analyze`.
|
||||
- filters analyze execution only; does not force stage rerun.
|
||||
- accepted on `run`, `resume`, `run-stage analyze`, and `analyze`.
|
||||
- filters analyze execution only.
|
||||
- does not imply force on `run`, `resume`, or `run-stage`; `narratio analyze` is force-by-design.
|
||||
|
||||
Canonical previous-session input behavior:
|
||||
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
|
||||
@@ -116,6 +144,7 @@ When archive is enabled and run upload is enabled, archive publishes under:
|
||||
Archive uploads:
|
||||
- run record files from run root (excluding `audio/`).
|
||||
- promoted files from explicit `archive.promote_artifacts` rules.
|
||||
- mutable session locks from helper commands live at `{session_prefix}/locks.yml`.
|
||||
|
||||
Publish order:
|
||||
1. upload `current/manifest.json`
|
||||
@@ -129,6 +158,17 @@ Archive promotion is explicit and source-based:
|
||||
- missing required promotion sources fail archive stage.
|
||||
- missing optional promotion sources are skipped.
|
||||
- invalid resolved artifacts fail archive stage.
|
||||
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
|
||||
- remote locks from `{session_prefix}/locks.yml` are merged with static `archive.locks`; static locks win on duplicate sources.
|
||||
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
|
||||
|
||||
Lock helper behavior:
|
||||
- `narratio locks --session-id <id>` lists effective static and remote locks.
|
||||
- `narratio locks add --session-id <id> --reason <text> <source>` writes a remote lock.
|
||||
- `narratio locks add --session-id <id> --force --reason <text> <source>` updates an existing remote lock reason.
|
||||
- `narratio locks remove --session-id <id> <source>` removes only a remote lock.
|
||||
- `locks remove` cannot remove static pipeline locks.
|
||||
- remote lock writes check whether the lock store exists, but are not compare-and-swap atomic.
|
||||
|
||||
## Resume, retry, restore, and safe rerun behavior
|
||||
|
||||
@@ -146,6 +186,7 @@ Restore conflict policy:
|
||||
|
||||
Forced reruns:
|
||||
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
||||
- ordinary `--force` does not override archive locks.
|
||||
|
||||
Safe rerun pattern:
|
||||
1. rerun the changed stage with `--force`.
|
||||
@@ -153,12 +194,20 @@ Safe rerun pattern:
|
||||
|
||||
## Cleanup behavior
|
||||
|
||||
Cleanup is considered only when archive stage executed and succeeded.
|
||||
Automatic post-archive cleanup is considered only when archive stage executed and succeeded.
|
||||
|
||||
Cleanup toggles:
|
||||
Automatic cleanup toggles:
|
||||
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
|
||||
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
|
||||
|
||||
Manual cleanup:
|
||||
- `narratio clean --session-id <id>` deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
|
||||
- `narratio clean --all` deletes all local session work under `{workspace.root}/work` and all spool children under `{spool.root}`.
|
||||
- `--dry-run` prints targets without deleting.
|
||||
- `--clear-cache` also removes matching S3 audio cache files. Without it, cache is preserved.
|
||||
|
||||
The S3 audio cache under `pipeline.cache.root` is durable input cache state, not workspace or spool state. Automatic cleanup and default manual cleanup do not delete it.
|
||||
|
||||
Cleanup eligibility gates:
|
||||
- archive enabled
|
||||
- archive run upload enabled
|
||||
@@ -185,18 +234,26 @@ Recommended recovery:
|
||||
|
||||
1. inspect state:
|
||||
|
||||
```bash
|
||||
narratio status --session-id 2026-04-04
|
||||
```
|
||||
|
||||
This reports local manifest state, committed remote current state, expected remote transcript/artifact availability, and archive locks.
|
||||
|
||||
2. for one manifest file, run:
|
||||
|
||||
```bash
|
||||
narratio status --manifest <manifest-path>
|
||||
```
|
||||
|
||||
2. for restore-specific checks, run:
|
||||
3. for restore-specific checks, run:
|
||||
|
||||
```bash
|
||||
narratio restore --session-id 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
3. fix root cause (config/input/credentials/storage/service availability).
|
||||
4. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
|
||||
4. fix root cause (config/input/credentials/storage/service availability).
|
||||
5. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
|
||||
|
||||
## Restore report
|
||||
|
||||
@@ -213,7 +270,9 @@ Dry-run does not write restore report files.
|
||||
|
||||
## Operational caveats
|
||||
|
||||
- `status` requires explicit `--manifest`; there is no session-id lookup command.
|
||||
- `status` with no config/session flags still requires explicit `--manifest`.
|
||||
- `status --session-id <id>` uses normal config/session loading, including remote session fallback.
|
||||
- `status --session-id <id>` includes the same promoted remote output availability view as `artifacts list --remote` when storage is configured.
|
||||
- local and S3 audio input modes are mutually exclusive.
|
||||
- archive publish requires upstream stages through `analyze` to be `succeeded`.
|
||||
- required promotion rules can fail when selected analyze artifacts did not generate a required file path.
|
||||
|
||||
52
docs/roadmap/operations.md
Normal file
52
docs/roadmap/operations.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Roadmap: Operator Helper Commands
|
||||
|
||||
## Status
|
||||
|
||||
Implemented.
|
||||
|
||||
The operator helper command set is no longer conceptual. Current behavior is documented in:
|
||||
|
||||
- `docs/cli.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/config.md`
|
||||
- `docs/internal/artifacts.md`
|
||||
- `docs/internal/stage-archive.md`
|
||||
|
||||
## Implemented Commands
|
||||
|
||||
- `narratio session validate`
|
||||
- `narratio status --manifest <path>`
|
||||
- `narratio status --session-id <id>`
|
||||
- `narratio session init --output <path>`
|
||||
- `narratio session init --remote`
|
||||
- `narratio artifacts list`
|
||||
- `narratio artifacts list --remote`
|
||||
- `narratio locks`
|
||||
- `narratio locks add <source>`
|
||||
- `narratio locks remove <source>`
|
||||
|
||||
## Implemented Decisions
|
||||
|
||||
- Helper output is text-only. No JSON schema exists yet.
|
||||
- `status` remains a top-level command.
|
||||
- `session validate`, `session init`, and `artifacts list` are nested helper commands.
|
||||
- `locks` is the single top-level command for listing, adding, and removing archive promotion locks.
|
||||
- Remote session initialization requires explicit `--remote`.
|
||||
- Local session initialization requires `--output`.
|
||||
- Remote artifact availability is opt-in with `artifacts list --remote`.
|
||||
- Mutable locks are source-based and stored at `{session_prefix}/locks.yml`.
|
||||
- The remote lock store uses strict YAML with top-level `locks`.
|
||||
- Static `pipeline.archive.locks` and remote locks are merged; static locks win on duplicate sources.
|
||||
- `locks remove` removes only remote locks.
|
||||
- Ordinary execution `--force` does not override locks.
|
||||
- Remote lock writes use existence checks and `--force` for updates; there is no compare-and-swap protection.
|
||||
|
||||
## Remaining Future Enhancements
|
||||
|
||||
These are intentionally not implemented:
|
||||
|
||||
- `--json` output for helper commands.
|
||||
- Optimistic concurrency or ETag compare-and-swap for remote lock mutations.
|
||||
- Rich remote artifact availability across historical run-local objects.
|
||||
- Session-lock acquisition for remote mutation helpers.
|
||||
- Broader campaign helper commands such as `campaign validate` or `campaign publish`.
|
||||
@@ -1,318 +0,0 @@
|
||||
# Roadmap: Campaign State, Remote Sessions, and Archive Locks
|
||||
|
||||
## Purpose
|
||||
|
||||
This roadmap describes planned, unimplemented work for three related feature areas:
|
||||
|
||||
1. `campaign.yml` configuration for stable campaign-level inputs.
|
||||
2. Remote `session.yml` loading from the existing S3 object-store backend.
|
||||
3. Logical archive locks that prevent selected top-level transcript/artifact promotions from overwriting curated archive state while still preserving run-local outputs.
|
||||
|
||||
Treat this document as an implementation plan, not as current behavior. Keep planned behavior under `docs/roadmap/` until each phase is implemented and canonical docs are updated.
|
||||
|
||||
## Current Code Facts
|
||||
|
||||
The current codebase already settles several design choices:
|
||||
|
||||
- CLI commands use short noun flags: `--config`, `--session`, `--session-id`, `--previous-session-id`, `--force`, and `--artifacts`.
|
||||
- `run-stage` uses flags before the positional stage name, for example:
|
||||
|
||||
narratio run-stage --session ./session.yml prepare
|
||||
|
||||
- `session.yml` is represented by `config.SessionConfig` and currently owns `session_id`, `previous_session_id`, `campaign`, `date`, `title`, and `inputs`.
|
||||
- Strict YAML decoding is already implemented with `yaml.Decoder.KnownFields(true)`.
|
||||
- Local session discovery is already ordered as `./session.yml`, `/usr/local/etc/narratio/session.yml`, then `/etc/narratio/session.yml`.
|
||||
- The canonical S3 session prefix is already:
|
||||
|
||||
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/
|
||||
|
||||
- Archive promotion is already source-based through `archive.promote_artifacts[].source`, with destination derivation and validation in `internal/config`.
|
||||
- Storage adapters receive bucket-relative keys and do not infer campaign, session, run, or root-prefix semantics.
|
||||
|
||||
## Guardrails
|
||||
|
||||
Keep Narratio explicit and stage-driven. Do not introduce a generic workflow engine, broad config language, or stage behavior that reaches through adapter boundaries.
|
||||
|
||||
Implementation must preserve these constraints:
|
||||
|
||||
- Keep storage details behind `internal/adapters/storage`.
|
||||
- Compute session, campaign, archive, and remote config keys in app/artifact/path helpers, not inside storage implementations.
|
||||
- Use centralized path helpers in `internal/artifacts` or the established local path model.
|
||||
- Preserve manifest-driven resume and stage status semantics.
|
||||
- Keep strict YAML decoding for `pipeline.yml`, `campaign.yml`, and `session.yml`.
|
||||
- Keep raw secrets out of configs, manifests, logs, generated configs, archive metadata, and roadmap examples.
|
||||
- Update canonical user-facing docs only after behavior is implemented.
|
||||
|
||||
## Phase 1: Add `campaign.yml`
|
||||
|
||||
Add campaign-level configuration for stable campaign identity and stable input files. Do not add remote campaign loading in this phase.
|
||||
|
||||
### CLI and Discovery
|
||||
|
||||
Add `--campaign <path>` to `run`, `plan`, `resume`, `run-stage`, and `restore`.
|
||||
|
||||
Examples:
|
||||
|
||||
narratio run --campaign ./campaign.yml --session ./session.yml
|
||||
narratio plan --campaign ./campaign.yml --session ./session.yml
|
||||
narratio resume --campaign ./campaign.yml --session ./session.yml
|
||||
narratio run-stage --campaign ./campaign.yml --session ./session.yml prepare
|
||||
narratio restore --campaign ./campaign.yml --session ./session.yml
|
||||
|
||||
Campaign config discovery order:
|
||||
|
||||
1. explicit `--campaign <path>`;
|
||||
2. `./campaign.yml`;
|
||||
3. `/usr/local/etc/narratio/campaign.yml`;
|
||||
4. `/etc/narratio/campaign.yml`.
|
||||
|
||||
Implement this in the same style as `resolvePipelineConfigPath` and `resolveSessionConfigPath`. Add default path constants and a search-path variable in `internal/config/defaults.go`.
|
||||
|
||||
### Config Shape
|
||||
|
||||
Initial `campaign.yml` fields:
|
||||
|
||||
campaign: icewind-dale
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
|
||||
Do not add speculative campaign artifact defaults, prompt defaults, or title conventions in the first implementation.
|
||||
|
||||
### Merge Behavior
|
||||
|
||||
Add `CampaignConfig` and keep the final stage-facing config explicit.
|
||||
|
||||
Required behavior:
|
||||
|
||||
- `pipeline.yml` remains host/runtime configuration.
|
||||
- `campaign.yml` supplies campaign identity and stable input file defaults.
|
||||
- `session.yml` remains the source for `session_id`, `previous_session_id`, `date`, `title`, and audio input.
|
||||
- Campaign-level `speakers_file`, `autocorrect_file`, and `glossary_file` fill missing session-level stable input fields.
|
||||
- Session-level stable input fields override campaign-level stable input fields.
|
||||
- If both `campaign.yml` and `session.yml` specify `campaign`, the values must match.
|
||||
- The resolved session must satisfy the existing session validation rules before stages run.
|
||||
- Unknown fields in `campaign.yml` fail strict decode.
|
||||
|
||||
Path resolution must preserve source-file locality:
|
||||
|
||||
- campaign-provided stable input paths resolve relative to `campaign.yml`;
|
||||
- session-provided stable input overrides resolve relative to `session.yml`;
|
||||
- absolute paths keep existing behavior.
|
||||
|
||||
Track enough provenance in the resolved config or prepare inputs so `prepare` can copy the correct source files without guessing which file supplied each path.
|
||||
|
||||
### Prepare Behavior
|
||||
|
||||
Update `prepare` to materialize the resolved campaign/session inputs into canonical session input paths:
|
||||
|
||||
inputs/campaign.yml
|
||||
inputs/session.yml
|
||||
inputs/pipeline.resolved.yml
|
||||
inputs/speakers.yml
|
||||
inputs/autocorrect.yml
|
||||
inputs/glossary.yml
|
||||
|
||||
Continue recording deterministic `manifest.Inputs` records with checksums. If a prepared input came from `campaign.yml`, record source/provenance using the existing manifest input fields where practical; add narrow metadata only if the existing fields cannot describe it.
|
||||
|
||||
## Phase 2: Load Remote `session.yml`
|
||||
|
||||
Support running with no local session file when a remote session file exists under the canonical session prefix.
|
||||
|
||||
### Preconditions
|
||||
|
||||
Build this phase after `campaign.yml`, because campaign identity is required to compute the remote session key. Do not infer campaign identity from object-store listing.
|
||||
|
||||
### Loading Precedence
|
||||
|
||||
Session loading order:
|
||||
|
||||
1. If `--session <path>` is supplied, load that local file.
|
||||
2. If `--session` is omitted, use existing local discovery: `./session.yml`, `/usr/local/etc/narratio/session.yml`, `/etc/narratio/session.yml`.
|
||||
3. If no local session file is found, `--session-id` is present, storage is configured, and campaign identity is resolved, load remote `session.yml`.
|
||||
4. If no local or remote session can be loaded, fail with a message that lists the local search paths and the remote key that was attempted when applicable.
|
||||
|
||||
Do not make remote loading mask local discovery. Existing local discovery remains the local fallback before remote is attempted. Once remote loading is attempted, a missing remote object, storage init error, or malformed remote YAML fails clearly because no local session was available.
|
||||
|
||||
### Remote Key Layout
|
||||
|
||||
Use the existing canonical S3 layout:
|
||||
|
||||
session prefix: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
|
||||
session file: {session_prefix}/session.yml
|
||||
audio prefix: {session_prefix}/{session.inputs.audio_s3.prefix}
|
||||
|
||||
Add a centralized helper near `internal/artifacts/s3_keys.go`:
|
||||
|
||||
S3SessionConfigKey(sessionPrefix string) string
|
||||
|
||||
The helper should return `{session_prefix}/session.yml` using the same key normalization style as `S3CurrentManifestKey`, `S3CurrentRunPointerKey`, and `S3PromotedArtifactKey`.
|
||||
|
||||
### Decode, Template, and Provenance
|
||||
|
||||
Remote `session.yml` uses the same template variables and mismatch checks as local sessions:
|
||||
|
||||
- `{{session_id}}`
|
||||
- `{{ session_id }}`
|
||||
- `{{previous_session_id}}`
|
||||
- `{{ previous_session_id }}`
|
||||
|
||||
Decode remote session YAML with strict known-field validation. Reuse the current session template/render/decode path by adding a byte/string-based loader rather than duplicating YAML decode logic.
|
||||
|
||||
When `prepare` materializes a remote session into `inputs/session.yml`, record that it came from S3. Preserve useful non-secret provenance when available:
|
||||
|
||||
- bucket;
|
||||
- key;
|
||||
- ETag;
|
||||
- size;
|
||||
- local checksum;
|
||||
- downloaded temp/materialized path.
|
||||
|
||||
## Phase 3: Add Logical Archive Locks
|
||||
|
||||
Add source-based archive locks under `pipeline.archive.locks`. The current promotion system is already source-based, so the first implementation must not support destination-based locks.
|
||||
|
||||
### Config Shape
|
||||
|
||||
Add lock entries:
|
||||
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.transcript.polished
|
||||
reason: Human-reviewed transcript; do not overwrite automatically.
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: Final recap was manually edited.
|
||||
|
||||
Validation rules:
|
||||
|
||||
- `source` is required.
|
||||
- `source` must be a built-in source ID or configured `narratio.artifact.<key>` accepted by the same source validation used for `promote_artifacts`.
|
||||
- `reason` is optional and non-secret.
|
||||
- duplicate lock sources fail validation.
|
||||
- lock entries do not support `dest` in the first implementation; unknown fields already fail strict decode.
|
||||
|
||||
### Archive Behavior
|
||||
|
||||
Archive must continue uploading complete run-local outputs under `runs/{run_id}/`.
|
||||
|
||||
Promotion behavior:
|
||||
|
||||
1. Resolve promotion source and destination using existing source-based promotion logic.
|
||||
2. If the promotion source is unlocked, upload the top-level promoted object normally.
|
||||
3. If the promotion source is locked, skip only the top-level promotion overwrite.
|
||||
4. Treat locked required promotions as intentional successful skips by default.
|
||||
5. Continue archive commit when all run-local uploads and all non-locked required promotions succeed.
|
||||
6. Upload `current/manifest.json` and `current/run_id.txt` in the existing order, with `current/run_id.txt` last.
|
||||
|
||||
Ordinary `--force` must not override locks. Do not implement a lock-break override in this phase.
|
||||
|
||||
### Metadata
|
||||
|
||||
Record locked promotion skips in archive metadata/reporting so operators can distinguish missing optional promotions from lock-protected promotions.
|
||||
|
||||
Include:
|
||||
|
||||
- source ID;
|
||||
- destination relative path and remote key;
|
||||
- reason;
|
||||
- local resolved path;
|
||||
- resolved provenance;
|
||||
- whether the original promotion rule was required.
|
||||
|
||||
Keep existing metadata such as `promoted_paths`, `skipped_optional_promotions`, `current_manifest_key`, `current_run_id_key`, and `current_pointer_written`.
|
||||
|
||||
## Phase 4: Future Operator Helpers
|
||||
|
||||
These commands are future work only. Do not implement them with the first campaign, remote-session, or lock changes.
|
||||
|
||||
Potential helper shapes:
|
||||
|
||||
narratio session validate --session-id 2026-06-07
|
||||
narratio session init --session-id 2026-06-07 --title "The Black Cabin"
|
||||
narratio status --session-id 2026-06-07
|
||||
narratio locks --session-id 2026-06-07
|
||||
narratio lock narratio.artifact.session_recap --session-id 2026-06-07
|
||||
narratio unlock narratio.artifact.session_recap --session-id 2026-06-07
|
||||
|
||||
Potential behavior:
|
||||
|
||||
- validate remote session config;
|
||||
- check audio object availability;
|
||||
- inspect committed remote current state;
|
||||
- list promoted transcripts/artifacts;
|
||||
- list archive lock status;
|
||||
- initialize a remote session skeleton;
|
||||
- publish or sync campaign assets.
|
||||
|
||||
## Implementation Sequence
|
||||
|
||||
Use small, reviewable commits.
|
||||
|
||||
1. Campaign config types and discovery:
|
||||
add `CampaignConfig`, strict loading, defaults/search paths, `--campaign` flags, and config/app tests.
|
||||
2. Campaign/session merge:
|
||||
implement resolved stable input merge, path provenance, validation, and prepare materialization.
|
||||
3. Campaign docs after implementation:
|
||||
update canonical docs and examples only for implemented behavior.
|
||||
4. Remote session key and loader:
|
||||
add `S3SessionConfigKey`, byte/string session loading, remote download through `ObjectStore`, and app-level precedence tests.
|
||||
5. Remote session prepare provenance:
|
||||
materialize downloaded session config and record S3 provenance.
|
||||
6. Remote session docs after implementation:
|
||||
update canonical docs and examples only after behavior exists.
|
||||
7. Archive lock config:
|
||||
add lock config structs, strict decode coverage, source validation, and duplicate detection.
|
||||
8. Archive lock enforcement:
|
||||
skip locked top-level promotions, preserve run-local uploads, record lock metadata, and protect commit ordering.
|
||||
9. Final sweep:
|
||||
run focused tests, then `go test ./...`; verify planned behavior remains only in roadmap docs until implemented.
|
||||
|
||||
## Test Plan
|
||||
|
||||
Add focused coverage in these packages:
|
||||
|
||||
- `internal/config`: campaign load, strict decode, discovery constants, merge validation, campaign/session mismatch, lock validation, duplicate lock rejection.
|
||||
- `internal/app`: `--campaign` parsing on `run`, `plan`, `resume`, `run-stage`, and `restore`; campaign discovery; explicit `--session` precedence; local discovery before remote; remote session fallback when local discovery misses.
|
||||
- `internal/artifacts`: `S3SessionConfigKey`; canonical session prefix compatibility; source ID validation for lock sources.
|
||||
- `internal/stage/prepare`: campaign/session stable input materialization; campaign-relative and session-relative path resolution; remote session provenance in `manifest.Inputs`.
|
||||
- `internal/stage/archive`: locked required promotion succeeds as skipped; unlocked promotion uploads; run-local outputs upload when top-level promotion is locked; `--force` does not break locks; `current/run_id.txt` remains the last upload.
|
||||
- `internal/adapters/storage`: fake object key normalization and remote session download expectations.
|
||||
|
||||
Run at least:
|
||||
|
||||
go test ./internal/config -v
|
||||
go test ./internal/app -run TestExecute -v
|
||||
go test ./internal/artifacts -v
|
||||
go test ./internal/stage -run 'Prepare|Archive' -v
|
||||
go test ./internal/adapters/storage -v
|
||||
go test ./...
|
||||
|
||||
Use fake storage for remote-session and archive-lock behavior. Ordinary tests must not require live S3.
|
||||
|
||||
## Documentation Updates After Implementation
|
||||
|
||||
After each phase is implemented, update only docs for behavior that exists.
|
||||
|
||||
Likely files:
|
||||
|
||||
- `docs/config.md`
|
||||
- `docs/cli.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/stage-prepare.md`
|
||||
- `docs/internal/stage-archive.md`
|
||||
- `docs/internal/storage.md`
|
||||
- `docs/internal/artifacts.md`
|
||||
- relevant examples under `examples/`
|
||||
|
||||
Do not document remote campaign loading, helper commands, or lock override flags as current behavior until implemented.
|
||||
|
||||
## Remaining Open Decisions
|
||||
|
||||
The codebase resolves the campaign flag name, campaign discovery order, remote session layout, local-vs-remote session precedence, source-based archive lock model, and locked required promotion policy.
|
||||
|
||||
Remaining decisions:
|
||||
|
||||
1. Whether `campaign.yml` should eventually be loadable from S3. Do not implement remote campaign loading in the first phase.
|
||||
2. Whether a future explicit lock override command or flag is needed. Do not make ordinary `--force` break locks.
|
||||
3. Whether future helper commands should be top-level commands or subcommands. Keep them out of the first implementation.
|
||||
@@ -9,19 +9,19 @@ Symptom:
|
||||
- `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found.
|
||||
|
||||
Likely Cause:
|
||||
- `pipeline.yml` or `session.yml` is missing from discovery paths.
|
||||
- wrong working directory when relying on `./session.yml`.
|
||||
- `pipeline.yml`, `campaign.yml`, or `session.yml` is missing from system discovery paths.
|
||||
- a local working-directory config file was not passed explicitly.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
pwd
|
||||
ls -l ./session.yml
|
||||
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
|
||||
ls -l /usr/local/etc/narratio/campaign.yml /etc/narratio/campaign.yml
|
||||
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
- pass explicit `--config` and `--session`.
|
||||
- pass explicit `--config`, `--campaign`, and `--session`.
|
||||
- or place files in documented discovery paths.
|
||||
|
||||
Links:
|
||||
@@ -40,7 +40,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio plan --session ./session.yml --session-id 2026-04-04
|
||||
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session ./session.yml --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -62,7 +62,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -84,7 +84,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio run --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
|
||||
narratio run --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -106,7 +106,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
|
||||
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -129,7 +129,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -154,7 +154,7 @@ Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio status --manifest /path/to/manifest.json
|
||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
|
||||
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -227,7 +227,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -250,7 +250,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -272,7 +272,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -342,7 +342,7 @@ Likely Cause:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
|
||||
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
@@ -366,7 +366,7 @@ Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio status --manifest /path/to/manifest.json
|
||||
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 archive
|
||||
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 archive
|
||||
```
|
||||
|
||||
Safe Fix:
|
||||
|
||||
5
examples/campaign.yml
Normal file
5
examples/campaign.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
@@ -1,9 +1,5 @@
|
||||
session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./examples/speakers.yml
|
||||
autocorrect_file: ./examples/autocorrect.yml
|
||||
glossary_file: ./examples/glossary.yml
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
speakers_file: ./examples/speakers.yml
|
||||
autocorrect_file: ./examples/autocorrect.yml
|
||||
glossary_file: ./examples/glossary.yml
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./examples/speakers.yml
|
||||
autocorrect_file: ./examples/autocorrect.yml
|
||||
glossary_file: ./examples/glossary.yml
|
||||
|
||||
@@ -27,8 +27,9 @@ type FakeBackend struct {
|
||||
Err error
|
||||
Result ArchiveResult
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
@@ -43,6 +44,12 @@ type FakeUploadCall struct {
|
||||
Options UploadOptions
|
||||
}
|
||||
|
||||
// FakeDownloadCall captures one download invocation in call order.
|
||||
type FakeDownloadCall struct {
|
||||
Key string
|
||||
LocalPath string
|
||||
}
|
||||
|
||||
// Archive records request and returns configured response.
|
||||
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -130,6 +137,10 @@ func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error
|
||||
if !ok {
|
||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||
}
|
||||
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||
Key: normalizeObjectKey(key),
|
||||
LocalPath: localPath,
|
||||
})
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||
|
||||
@@ -56,7 +56,7 @@ func TestFakeBackendDownload(t *testing.T) {
|
||||
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
|
||||
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(dst)
|
||||
|
||||
@@ -9,17 +9,19 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
||||
[]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
@@ -33,12 +35,12 @@ func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
||||
|
||||
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
[]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
@@ -52,7 +54,7 @@ func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||
|
||||
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -65,7 +67,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
||||
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -78,7 +80,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||
|
||||
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -93,7 +95,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := Resume(
|
||||
context.Background(),
|
||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -104,10 +106,154 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string) {
|
||||
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedForce = opts.Force
|
||||
return &RunSummary{
|
||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||
Executed: []string{"analyze"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
|
||||
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
|
||||
}
|
||||
if !capturedForce {
|
||||
t.Fatal("captured force = false, want true")
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
|
||||
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{
|
||||
"analyze",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--artifacts", "player_handout,session_recap",
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "positional", args: []string{"analyze", "extra"}, want: "analyze: unexpected positional arguments"},
|
||||
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"analyze"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsageIncludesAnalyze(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(nil, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze") {
|
||||
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("open pipeline config for append: %v", err)
|
||||
@@ -136,5 +282,5 @@ scriptorium:
|
||||
if _, err := f.WriteString(extra); err != nil {
|
||||
t.Fatalf("append scriptorium config: %v", err)
|
||||
}
|
||||
return pipelinePath, sessionPath
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
49
internal/app/campaign_config_path.go
Normal file
49
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func resolveCampaignConfigPath(flagValue string) (string, error) {
|
||||
return resolveCampaignConfigPathWithCandidates(flagValue, config.DefaultCampaignConfigSearchPaths)
|
||||
}
|
||||
|
||||
func resolveCampaignConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||
return explicit, nil
|
||||
}
|
||||
|
||||
ordered := make([]string, 0, len(candidates))
|
||||
for _, raw := range candidates {
|
||||
path := strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
ordered = append(ordered, path)
|
||||
info, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if info.IsDir() {
|
||||
continue
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return "", fmt.Errorf("check default campaign config %q: %w", path, err)
|
||||
}
|
||||
|
||||
if len(ordered) == 0 {
|
||||
return "", fmt.Errorf("no campaign config path provided and no default locations configured")
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"no campaign config path provided and no default campaign config found; searched: %s; pass --campaign to use an explicit path",
|
||||
strings.Join(ordered, ", "),
|
||||
)
|
||||
}
|
||||
49
internal/app/campaign_config_path_test.go
Normal file
49
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveCampaignConfigPathExplicitWins(t *testing.T) {
|
||||
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||
got, err := resolveCampaignConfigPathWithCandidates(explicit, []string{filepath.Join(t.TempDir(), "campaign.yml")})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||
}
|
||||
if got != explicit {
|
||||
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathUsesFirstExistingDefault(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.yml")
|
||||
found := filepath.Join(dir, "campaign.yml")
|
||||
if err := os.WriteFile(found, []byte("campaign: sample-campaign\n"), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
|
||||
got, err := resolveCampaignConfigPathWithCandidates("", []string{missing, found})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||
}
|
||||
if got != filepath.Clean(found) {
|
||||
t.Fatalf("path = %q, want %q", got, filepath.Clean(found))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||
_, err := resolveCampaignConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/campaign.yml", "/etc/narratio/campaign.yml"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "searched") {
|
||||
t.Fatalf("error = %q, want searched paths", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pass --campaign") {
|
||||
t.Fatalf("error = %q, want explicit-campaign guidance", err.Error())
|
||||
}
|
||||
}
|
||||
336
internal/app/clean.go
Normal file
336
internal/app/clean.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
// state unless cache cleanup is explicitly requested.
|
||||
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var all bool
|
||||
var dryRun bool
|
||||
var clearCache bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
||||
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
||||
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("clean: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("clean: unexpected positional arguments")
|
||||
}
|
||||
if all {
|
||||
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||
}
|
||||
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||
}
|
||||
|
||||
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: --session-id is required unless --all is set")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||
}
|
||||
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||
if campaign == "" || sessionID == "" {
|
||||
return fmt.Errorf("clean: campaign and session_id are required")
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
||||
}
|
||||
|
||||
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
||||
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||
strings.TrimSpace(flags.sessionID) != "" ||
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --session, --session-id, or --previous-session-id")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintln(out, "Clean plan for all local sessions")
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cleaned all local sessions")
|
||||
}
|
||||
|
||||
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
||||
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dryRun {
|
||||
if dir.Exists {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !dir.Exists {
|
||||
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
||||
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(entry); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
if cleanRoot == "" {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
info, err := os.Lstat(rootAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return rootAbs, nil, nil
|
||||
}
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
|
||||
}
|
||||
entries, err := os.ReadDir(rootAbs)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
out := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(rootAbs, entry.Name())
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
||||
}
|
||||
out = append(out, path)
|
||||
}
|
||||
return rootAbs, out, nil
|
||||
}
|
||||
|
||||
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||
if cfg.Session.Inputs.AudioS3 == nil {
|
||||
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||
return nil
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := store.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
count := 0
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||
continue
|
||||
}
|
||||
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if deleted {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
||||
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
||||
}
|
||||
|
||||
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
||||
file, err := validateScopedFile(root, target, policy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if dryRun {
|
||||
if file.Exists {
|
||||
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if !file.Exists {
|
||||
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if err := os.Remove(file.TargetAbs); err != nil {
|
||||
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
if cleanRoot == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
if cleanTarget == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||
}
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
targetAbs, err := filepath.Abs(cleanTarget)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||
}
|
||||
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||
}
|
||||
if rel == "." {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||
}
|
||||
info, err := os.Lstat(targetAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||
}
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
|
||||
}
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||
}
|
||||
|
||||
func cleanIsFlac(path string) bool {
|
||||
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||
}
|
||||
255
internal/app/clean_test.go
Normal file
255
internal/app/clean_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workDir)
|
||||
cleanAssertMissing(t, spoolDir)
|
||||
cleanAssertExists(t, cachePath)
|
||||
if !strings.Contains(stdout.String(), "Cache: preserved") {
|
||||
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--dry-run"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertExists(t, workDir)
|
||||
cleanAssertExists(t, spoolDir)
|
||||
if !strings.Contains(stdout.String(), "Would delete:") {
|
||||
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Missing:") {
|
||||
t.Fatalf("stdout = %q, want missing path output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write session: %v", err)
|
||||
}
|
||||
|
||||
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, cachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
|
||||
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
workRoot := filepath.Join(workspaceRoot, "work")
|
||||
spoolRoot := filepath.Join(workspaceRoot, "spool")
|
||||
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workRoot)
|
||||
cleanAssertExists(t, spoolRoot)
|
||||
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
|
||||
cleanAssertExists(t, cachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, audioCachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, audioCachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--all"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--all cannot be combined") {
|
||||
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRequiresSessionID(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--session-id is required unless --all is set") {
|
||||
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRejectsUnsafeTargets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
|
||||
t.Fatal("outside target error = nil, want error")
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
|
||||
t.Fatal("root target error = nil, want error")
|
||||
}
|
||||
filePath := filepath.Join(root, "file.txt")
|
||||
mustWriteTestFile(t, filePath, "file")
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
|
||||
t.Fatal("file target error = nil, want error")
|
||||
}
|
||||
symlinkPath := filepath.Join(root, "link")
|
||||
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
|
||||
t.Fatal("symlink target error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearIsNotCommandAlias(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clear"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
|
||||
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected %q to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "restore"}
|
||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "analyze", "restore", "session", "artifacts", "locks", "clean"}
|
||||
|
||||
// Execute dispatches CLI commands and returns a process exit code.
|
||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
@@ -32,8 +32,18 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
err = Resume(ctx, cmdArgs, stdout)
|
||||
case "run-stage":
|
||||
err = RunStage(ctx, cmdArgs, stdout)
|
||||
case "analyze":
|
||||
err = Analyze(ctx, cmdArgs, stdout)
|
||||
case "restore":
|
||||
err = Restore(ctx, cmdArgs, stdout)
|
||||
case "session":
|
||||
err = Session(ctx, cmdArgs, stdout)
|
||||
case "artifacts":
|
||||
err = Artifacts(ctx, cmdArgs, stdout)
|
||||
case "locks":
|
||||
err = Locks(ctx, cmdArgs, stdout)
|
||||
case "clean":
|
||||
err = Clean(ctx, cmdArgs, stdout)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||
printUsage(stderr)
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
manifestPath := writeManifestPathForExecute(t)
|
||||
|
||||
cases := []struct {
|
||||
@@ -32,11 +32,11 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -94,12 +94,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
|
||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -110,14 +110,14 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
|
||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -136,19 +136,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
|
||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -188,6 +188,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
@@ -239,7 +240,7 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -252,6 +253,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
@@ -288,7 +290,7 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -305,11 +307,14 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||
config.DefaultCampaignConfigSearchPaths = []string{campaignPath}
|
||||
defer func() {
|
||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||
}()
|
||||
|
||||
var stdout bytes.Buffer
|
||||
@@ -323,6 +328,33 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMissingCampaignConfigReportsSearchedPaths(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
missingCampaignPath := filepath.Join(t.TempDir(), "campaign.yml")
|
||||
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||
config.DefaultCampaignConfigSearchPaths = []string{missingCampaignPath}
|
||||
defer func() {
|
||||
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||
}()
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "no campaign config path provided and no default campaign config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "pass --campaign") {
|
||||
t.Fatalf("stderr = %q, want explicit campaign guidance", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteInvalidCommand(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -359,11 +391,12 @@ func TestExecuteMissingCommand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
url := "https://example.com/transcribe"
|
||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||
@@ -377,6 +410,10 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
cache:
|
||||
root: ` + filepath.Join(workspaceRoot, "cache") + `
|
||||
spool:
|
||||
root: ` + filepath.Join(workspaceRoot, "spool") + `
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
@@ -407,9 +444,11 @@ notification:
|
||||
`
|
||||
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`
|
||||
campaignYAML := `campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
@@ -418,6 +457,9 @@ inputs:
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline config: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign config: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session config: %v", err)
|
||||
}
|
||||
@@ -427,7 +469,22 @@ inputs:
|
||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, sessionPath
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
campaignYAML := `campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
return campaignPath
|
||||
}
|
||||
|
||||
func writeManifestPathForExecute(t *testing.T) string {
|
||||
|
||||
133
internal/app/config_loader.go
Normal file
133
internal/app/config_loader.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvedCampaignPath, err := resolveCampaignConfigPath(campaignFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, explicitSession, sessionOpts)
|
||||
}
|
||||
|
||||
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if discoveredSession.Path != "" {
|
||||
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, discoveredSession.Path, sessionOpts)
|
||||
}
|
||||
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||
if sessionID == "" {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires --session-id")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
partialCfg := &config.Config{
|
||||
Pipeline: pipelineCfg,
|
||||
Campaign: campaignCfg,
|
||||
PipelinePath: resolvedPipelinePath,
|
||||
CampaignPath: resolvedCampaignPath,
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
|
||||
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(pipelineCfg)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config.Resolve(
|
||||
resolvedPipelinePath,
|
||||
pipelineCfg,
|
||||
resolvedCampaignPath,
|
||||
campaignCfg,
|
||||
sessionTempPath,
|
||||
sessionCfg,
|
||||
config.SessionSource{
|
||||
Source: "session_config.s3",
|
||||
LocalPath: sessionTempPath,
|
||||
S3Bucket: s3BucketName(pipelineCfg),
|
||||
S3Key: remoteKey,
|
||||
S3Size: sessionInfo.Size,
|
||||
S3ETag: sessionInfo.ETag,
|
||||
SpoolPath: sessionTempPath,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||
objects, err := store.List(ctx, sessionPrefix)
|
||||
if err != nil {
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == remoteKey {
|
||||
return obj, nil
|
||||
}
|
||||
}
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||
}
|
||||
|
||||
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
|
||||
f, err := os.CreateTemp("", "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
path := f.Name()
|
||||
if err := f.Close(); err != nil {
|
||||
return "", fmt.Errorf("close temp file %q: %w", path, err)
|
||||
}
|
||||
if err := store.Download(ctx, remoteKey, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
|
||||
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||
if cfg == nil || cfg.Storage.S3 == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||
}
|
||||
21
internal/app/object_store.go
Normal file
21
internal/app/object_store.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||
}
|
||||
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
165
internal/app/object_store_test.go
Normal file
165
internal/app/object_store_test.go
Normal file
@@ -0,0 +1,165 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
fake := &storage.FakeBackend{}
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||
return nil, errors.New("access key was not loaded before object store init")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||
return nil, errors.New("secret key was not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
if store != fake {
|
||||
t.Fatalf("store = %#v, want fake backend", store)
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("object store factory was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||
t.Setenv(secretKeyEnv, "existing-secret")
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||
return nil, errors.New("existing access key was overwritten")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||
return nil, errors.New("existing secret key was overwritten")
|
||||
}
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("object store factory was called after secret load failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig("")
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("factory boom")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
|
||||
t.Fatalf("error = %q, want factory context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{
|
||||
Backend: "s3",
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.TrimSpace(secretsDir) != "" {
|
||||
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||
t.Helper()
|
||||
originals := make(map[string]string, len(names))
|
||||
present := make(map[string]bool, len(names))
|
||||
for _, name := range names {
|
||||
value, ok := os.LookupEnv(name)
|
||||
originals[name] = value
|
||||
present[name] = ok
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, name := range names {
|
||||
if present[name] {
|
||||
_ = os.Setenv(name, originals[name])
|
||||
} else {
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
958
internal/app/operator_helpers.go
Normal file
958
internal/app/operator_helpers.go
Normal file
@@ -0,0 +1,958 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type commonConfigFlags struct {
|
||||
pipelinePath string
|
||||
campaignPath string
|
||||
sessionPath string
|
||||
sessionID string
|
||||
previousSessionID string
|
||||
}
|
||||
|
||||
type finding struct {
|
||||
Severity string
|
||||
Category string
|
||||
Message string
|
||||
}
|
||||
|
||||
type findingError struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (e findingError) Error() string {
|
||||
return fmt.Sprintf("%d validation error(s)", e.count)
|
||||
}
|
||||
|
||||
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&flags.campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
}
|
||||
|
||||
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||
return config.SessionLoadOptions{
|
||||
SessionID: f.sessionID,
|
||||
PreviousSessionID: f.previousSessionID,
|
||||
}
|
||||
}
|
||||
|
||||
// Session dispatches session helper subcommands.
|
||||
func Session(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("session: expected subcommand: validate|init")
|
||||
}
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return SessionValidate(ctx, args[1:], out)
|
||||
case "init":
|
||||
return SessionInit(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("session: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Artifacts dispatches artifact helper subcommands.
|
||||
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("artifacts: expected subcommand: list")
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
return ArtifactsList(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// SessionValidate performs a read-only session preflight.
|
||||
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("session validate: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("session validate: unexpected positional arguments")
|
||||
}
|
||||
|
||||
findings := []finding{}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
return renderFindings(out, "", "", findings)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
} else {
|
||||
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
|
||||
}
|
||||
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
|
||||
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
findings = append(findings, validateStableInputFindings(cfg)...)
|
||||
findings = append(findings, validateLocalAudioFindings(cfg)...)
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
if storeErr != nil {
|
||||
findings = append(findings, errorFinding("storage", storeErr.Error()))
|
||||
}
|
||||
if cfg.Session.Inputs.AudioS3 != nil {
|
||||
if storeErr != nil {
|
||||
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
|
||||
} else {
|
||||
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
|
||||
}
|
||||
}
|
||||
|
||||
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||
if len(requirements) == 0 {
|
||||
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||
} else if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
||||
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
||||
} else if storeErr != nil {
|
||||
findings = append(findings, errorFinding("previous", "previous-session artifacts cannot be checked because storage is unavailable"))
|
||||
} else {
|
||||
findings = append(findings, validatePreviousArtifactFindings(ctx, cfg, store, requirements)...)
|
||||
}
|
||||
|
||||
locks, lockErr := loadEffectiveLocks(ctx, cfg, store)
|
||||
if lockErr != nil {
|
||||
findings = append(findings, errorFinding("locks", lockErr.Error()))
|
||||
} else if len(locks.All) == 0 {
|
||||
findings = append(findings, okFinding("locks", "no effective archive locks"))
|
||||
} else {
|
||||
for _, lock := range locks.All {
|
||||
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
|
||||
}
|
||||
}
|
||||
if paths.ManifestPath != "" {
|
||||
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
|
||||
}
|
||||
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||
}
|
||||
|
||||
// Status reports either a requested manifest or effective local/remote session state.
|
||||
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var manifestPath string
|
||||
var flags commonConfigFlags
|
||||
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("status: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("status: unexpected positional arguments")
|
||||
}
|
||||
if strings.TrimSpace(manifestPath) != "" {
|
||||
return statusManifest(ctx, manifestPath, out)
|
||||
}
|
||||
if flags.pipelinePath == "" && flags.campaignPath == "" && flags.sessionPath == "" && flags.sessionID == "" && flags.previousSessionID == "" {
|
||||
return fmt.Errorf("status: --manifest is required")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
|
||||
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
|
||||
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
|
||||
|
||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||
} else if m == nil {
|
||||
fmt.Fprintln(out, "Local manifest: missing")
|
||||
} else {
|
||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||
writeStageStatuses(out, m)
|
||||
}
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
if storeErr != nil {
|
||||
fmt.Fprintf(out, "Remote archive: unavailable: %v\n", storeErr)
|
||||
} else if store != nil {
|
||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "Remote archive: missing or unavailable: %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Remote archive: current run %s\n", current.RunID)
|
||||
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
|
||||
}
|
||||
}
|
||||
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||
} else if storeErr == nil {
|
||||
catalogLocks := locks
|
||||
if err != nil {
|
||||
catalogLocks = &effectiveLocks{
|
||||
Static: staticArchiveLocks(cfg),
|
||||
All: staticArchiveLocks(cfg),
|
||||
}
|
||||
}
|
||||
promotedRemoteState := map[string]string{}
|
||||
if store != nil {
|
||||
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
||||
}
|
||||
fmt.Fprintln(out, "Remote outputs:")
|
||||
writeArtifactList(out, cfg, catalog, catalogLocks, promotedRemoteState)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "Archive locks: error: %v\n", err)
|
||||
} else {
|
||||
writeLocks(out, cfg, locks)
|
||||
}
|
||||
fmt.Fprintln(out, "Next actions:")
|
||||
fmt.Fprintf(out, "- narratio session validate --session-id %s\n", cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "- narratio restore --session-id %s --dry-run\n", cfg.Session.SessionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func statusManifest(ctx context.Context, manifestPath string, out io.Writer) error {
|
||||
store := &manifest.LocalStore{}
|
||||
m, err := store.Load(ctx, manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
|
||||
return err
|
||||
}
|
||||
writeStageStatuses(out, m)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SessionInit creates a local or remote session.yml skeleton.
|
||||
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var pipelinePath, campaignPath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||
var remote, force bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
|
||||
fs.StringVar(&date, "date", "", "session date")
|
||||
fs.StringVar(&title, "title", "", "session title")
|
||||
fs.StringVar(&output, "output", "", "local output session.yml path")
|
||||
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
|
||||
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("session init: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("session init: unexpected positional arguments")
|
||||
}
|
||||
if strings.TrimSpace(pipelinePath) == "" || strings.TrimSpace(campaignPath) == "" || strings.TrimSpace(sessionID) == "" {
|
||||
return fmt.Errorf("session init: --config, --campaign, and --session-id are required")
|
||||
}
|
||||
if (strings.TrimSpace(output) == "") == !remote {
|
||||
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
|
||||
}
|
||||
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
|
||||
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||
}
|
||||
|
||||
resolvedPipeline, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
resolvedCampaign, err := resolveCampaignConfigPath(campaignPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipeline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
campaignCfg, err := config.LoadCampaign(resolvedCampaign)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
data, err := buildSessionYAML(campaignCfg.Campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
label := strings.TrimSpace(output)
|
||||
if label == "" {
|
||||
label = "remote session.yml"
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
cfg, err := config.Resolve(resolvedPipeline, pipelineCfg, resolvedCampaign, campaignCfg, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
if !remote {
|
||||
if err := writeLocalFile(output, data, force); err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
|
||||
return err
|
||||
}
|
||||
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
|
||||
key := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
exists, err := store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: check remote session %q: %w", key, err)
|
||||
}
|
||||
if exists && !force {
|
||||
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
|
||||
}
|
||||
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("session init: write temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("session init: close temp file: %w", err)
|
||||
}
|
||||
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(pipelineCfg), key)
|
||||
return err
|
||||
}
|
||||
|
||||
// ArtifactsList lists effective artifact sources.
|
||||
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var remote bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&remote, "remote", false, "inspect remote archive availability")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("artifacts list: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("artifacts list: unexpected positional arguments")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
promotedRemoteState := map[string]string{}
|
||||
if remote && store != nil {
|
||||
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
||||
}
|
||||
writeArtifactList(out, cfg, catalog, locks, promotedRemoteState)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Locks dispatches archive lock list and mutation helpers.
|
||||
func Locks(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return LocksAdd(ctx, args[1:], out)
|
||||
case "remove":
|
||||
return LocksRemove(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("locks: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
return LocksList(ctx, args, out)
|
||||
}
|
||||
|
||||
// LocksList lists effective archive locks.
|
||||
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("locks: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("locks: unexpected positional arguments")
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks: --session-id is required")
|
||||
}
|
||||
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks: %w", err)
|
||||
}
|
||||
writeLocks(out, cfg, locks)
|
||||
return nil
|
||||
}
|
||||
|
||||
// LocksAdd adds or updates one remote lock.
|
||||
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var reason string
|
||||
var force bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.StringVar(&reason, "reason", "", "lock reason")
|
||||
fs.BoolVar(&force, "force", false, "update existing remote lock")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("locks add: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 1 {
|
||||
return fmt.Errorf("locks add: expected exactly one source id")
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks add: --session-id is required")
|
||||
}
|
||||
source := strings.TrimSpace(fs.Arg(0))
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, exists := remoteSet[source]; exists && !force {
|
||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
remoteSet[source] = config.ArchiveLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if _, err := config.ValidateArchiveLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio locks add: locked %s\n", source)
|
||||
return err
|
||||
}
|
||||
|
||||
// LocksRemove removes one remote lock.
|
||||
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("locks remove: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 1 {
|
||||
return fmt.Errorf("locks remove: expected exactly one source id")
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks remove: --session-id is required")
|
||||
}
|
||||
source := strings.TrimSpace(fs.Arg(0))
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, ok := remoteSet[source]; !ok {
|
||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||
}
|
||||
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
||||
}
|
||||
delete(remoteSet, source)
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio locks remove: unlocked %s\n", source)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
var store storage.ObjectStore
|
||||
if needStore {
|
||||
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||
}
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
return cfg, store, locks, m, nil
|
||||
}
|
||||
|
||||
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
|
||||
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
|
||||
date = strings.TrimSpace(sessionID)
|
||||
}
|
||||
type audioS3 struct {
|
||||
Prefix string `yaml:"prefix"`
|
||||
}
|
||||
type inputs struct {
|
||||
AudioDir string `yaml:"audio_dir,omitempty"`
|
||||
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
|
||||
}
|
||||
type sessionYAML struct {
|
||||
Campaign string `yaml:"campaign"`
|
||||
SessionID string `yaml:"session_id"`
|
||||
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
|
||||
Date string `yaml:"date,omitempty"`
|
||||
Title string `yaml:"title,omitempty"`
|
||||
Inputs inputs `yaml:"inputs"`
|
||||
}
|
||||
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
|
||||
if in.AudioDir == "" {
|
||||
prefix := strings.TrimSpace(audioS3Prefix)
|
||||
if prefix == "" {
|
||||
prefix = "audio/"
|
||||
}
|
||||
in.AudioS3 = &audioS3{Prefix: prefix}
|
||||
}
|
||||
data, err := yaml.Marshal(sessionYAML{
|
||||
Campaign: strings.TrimSpace(campaign),
|
||||
SessionID: strings.TrimSpace(sessionID),
|
||||
PreviousSessionID: strings.TrimSpace(previousSessionID),
|
||||
Date: strings.TrimSpace(date),
|
||||
Title: strings.TrimSpace(title),
|
||||
Inputs: in,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
|
||||
if campaign != "" || sessionID != "" {
|
||||
fmt.Fprintf(out, "Campaign: %s\n", campaign)
|
||||
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
|
||||
}
|
||||
errorsCount := 0
|
||||
for _, f := range findings {
|
||||
if f.Severity == "ERROR" {
|
||||
errorsCount++
|
||||
}
|
||||
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
|
||||
}
|
||||
if errorsCount > 0 {
|
||||
return findingError{count: errorsCount}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
|
||||
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
|
||||
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
|
||||
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
|
||||
|
||||
func sessionSourceSummary(cfg *config.Config) string {
|
||||
source := cfg.SessionSource.Source
|
||||
if source == "" {
|
||||
source = "session_config"
|
||||
}
|
||||
if cfg.SessionSource.S3Key != "" {
|
||||
return source + " " + cfg.SessionSource.S3Key
|
||||
}
|
||||
return source + " " + cfg.SessionPath
|
||||
}
|
||||
|
||||
func validateStableInputFindings(cfg *config.Config) []finding {
|
||||
items := []struct {
|
||||
name string
|
||||
in config.ResolvedInputFile
|
||||
}{
|
||||
{"speakers", cfg.StableInputs.SpeakersFile},
|
||||
{"autocorrect", cfg.StableInputs.AutocorrectFile},
|
||||
{"glossary", cfg.StableInputs.GlossaryFile},
|
||||
}
|
||||
out := make([]finding, 0, len(items))
|
||||
for _, item := range items {
|
||||
path, err := resolveHelperConfigRelativePath(item.in)
|
||||
if err != nil {
|
||||
out = append(out, errorFinding("inputs", item.name+": "+err.Error()))
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
out = append(out, errorFinding("inputs", fmt.Sprintf("%s missing: %v", item.name, err)))
|
||||
} else {
|
||||
out = append(out, okFinding("inputs", item.name+": "+path))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||
if strings.TrimSpace(input.ConfigPath) == "" {
|
||||
return "", fmt.Errorf("source config path is required")
|
||||
}
|
||||
path := strings.TrimSpace(input.Path)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("path is required")
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
|
||||
}
|
||||
|
||||
func validateLocalAudioFindings(cfg *config.Config) []finding {
|
||||
if cfg.Session.Inputs.AudioS3 != nil {
|
||||
return nil
|
||||
}
|
||||
audioDir := strings.TrimSpace(cfg.Session.Inputs.AudioDir)
|
||||
if audioDir == "" && len(cfg.Session.Inputs.AudioFiles) == 0 {
|
||||
return []finding{errorFinding("audio", "audio_dir, audio_files, or audio_s3 is required")}
|
||||
}
|
||||
base := filepath.Dir(cfg.SessionPath)
|
||||
paths := []string{}
|
||||
if audioDir != "" {
|
||||
dir := audioDir
|
||||
if !filepath.IsAbs(dir) {
|
||||
dir = filepath.Join(base, dir)
|
||||
}
|
||||
matches, err := filepath.Glob(filepath.Join(dir, "*.flac"))
|
||||
if err != nil || len(matches) == 0 {
|
||||
return []finding{errorFinding("audio", "no .flac files found in "+dir)}
|
||||
}
|
||||
paths = append(paths, matches...)
|
||||
}
|
||||
for _, file := range cfg.Session.Inputs.AudioFiles {
|
||||
p := file
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(base, p)
|
||||
}
|
||||
paths = append(paths, p)
|
||||
}
|
||||
for _, p := range paths {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return []finding{errorFinding("audio", fmt.Sprintf("audio file missing: %v", err))}
|
||||
}
|
||||
}
|
||||
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(paths)))}
|
||||
}
|
||||
|
||||
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := store.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return errorFinding("audio", err.Error())
|
||||
}
|
||||
count := 0
|
||||
for _, obj := range objects {
|
||||
if strings.HasSuffix(strings.ToLower(obj.Key), ".flac") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return errorFinding("audio", "no remote .flac objects found under "+audioPrefix)
|
||||
}
|
||||
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", count))
|
||||
}
|
||||
|
||||
func validatePreviousArtifactFindings(ctx context.Context, cfg *config.Config, store storage.ObjectStore, requirements []artifacts.PreviousArtifactRequirement) []finding {
|
||||
out := []finding{}
|
||||
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
||||
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(prefix)
|
||||
for _, key := range []string{runIDKey, manifestKey} {
|
||||
exists, err := store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
out = append(out, errorFinding("previous", fmt.Sprintf("check %s: %v", key, err)))
|
||||
return out
|
||||
}
|
||||
if !exists {
|
||||
out = append(out, errorFinding("previous", "missing "+key))
|
||||
return out
|
||||
}
|
||||
}
|
||||
for _, req := range requirements {
|
||||
out = append(out, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
return store.Load(ctx, path)
|
||||
}
|
||||
|
||||
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
|
||||
if m == nil || len(m.Stages) == 0 {
|
||||
fmt.Fprintln(out, "stages: no stages recorded")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(out, "stages:")
|
||||
names := make([]string, 0, len(m.Stages))
|
||||
for name := range m.Stages {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
||||
if cfg.Pipeline.Scriptorium != nil {
|
||||
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
||||
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
||||
}
|
||||
}
|
||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, promotedRemoteState map[string]string) {
|
||||
lockSet := lockSourceSet(locks.All)
|
||||
fmt.Fprintln(out, "Built-in:")
|
||||
for _, id := range []string{
|
||||
artifacts.ArtifactTranscriptMerged,
|
||||
artifacts.ArtifactTranscriptPolished,
|
||||
artifacts.ArtifactTranscriptFull,
|
||||
artifacts.ArtifactTranscriptTrimmed,
|
||||
artifacts.ArtifactBoundsSession,
|
||||
} {
|
||||
writeArtifactLine(out, id, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Configured:")
|
||||
for _, entry := range catalog.ListConfigured() {
|
||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Previous-session:")
|
||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
|
||||
}
|
||||
fmt.Fprintln(out, "Promoted:")
|
||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||
writePromotedArtifactLine(out, rule, catalog, lockSet, promotedRemoteState)
|
||||
}
|
||||
}
|
||||
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.ArchiveLockRule) {
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.ArchiveLockRule, remoteState map[string]string) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
dest, showDest, err := helperPromotionDest(rule, catalog)
|
||||
if err != nil {
|
||||
parts = append(parts, "remote=error")
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
return
|
||||
}
|
||||
if showDest {
|
||||
parts = append(parts, "dest="+dest)
|
||||
}
|
||||
if state := remoteState[promotionRemoteStateKey(source, dest)]; state != "" {
|
||||
parts = append(parts, state)
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func remotePromotionAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
out := map[string]string{}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest, _, err := helperPromotionDest(rule, catalog)
|
||||
if err != nil {
|
||||
out[promotionRemoteStateKey(source, "")] = "remote=error"
|
||||
continue
|
||||
}
|
||||
key := artifacts.S3PromotedArtifactKey(sessionPrefix, dest)
|
||||
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||
out[promotionRemoteStateKey(source, dest)] = "remote=promoted"
|
||||
} else if err != nil {
|
||||
out[promotionRemoteStateKey(source, dest)] = "remote=error"
|
||||
} else {
|
||||
out[promotionRemoteStateKey(source, dest)] = "remote=missing"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func helperPromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest := strings.TrimSpace(rule.Dest)
|
||||
if dest == "" {
|
||||
entry, ok := catalog.Lookup(source)
|
||||
if !ok {
|
||||
return "", false, fmt.Errorf("destination omitted and source is unknown")
|
||||
}
|
||||
dest = strings.TrimSpace(entry.CanonicalRelPath)
|
||||
if dest == "" {
|
||||
return "", false, fmt.Errorf("destination omitted and no canonical destination is available")
|
||||
}
|
||||
}
|
||||
normalized, err := normalizeHelperArchiveRelativePath(dest)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
entry, ok := catalog.Lookup(source)
|
||||
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
|
||||
return normalized, showDest, nil
|
||||
}
|
||||
|
||||
func normalizeHelperArchiveRelativePath(rel string) (string, error) {
|
||||
trimmed := strings.TrimSpace(rel)
|
||||
if trimmed == "" {
|
||||
return "", fmt.Errorf("relative path is required")
|
||||
}
|
||||
cleaned := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
|
||||
if cleaned == "." || cleaned == "" {
|
||||
return "", fmt.Errorf("relative path is required")
|
||||
}
|
||||
if filepath.IsAbs(trimmed) || strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
|
||||
return "", fmt.Errorf("path must be a clean relative path")
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func promotionRemoteStateKey(source, dest string) string {
|
||||
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
||||
}
|
||||
|
||||
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||
if locks == nil || len(locks.All) == 0 {
|
||||
fmt.Fprintln(out, "Archive locks: none")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(out, "Archive locks:")
|
||||
promoted := map[string]config.ArchivePromotionRule{}
|
||||
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Archive != nil {
|
||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||
promoted[strings.TrimSpace(rule.Source)] = rule
|
||||
}
|
||||
}
|
||||
staticSet := lockSourceSet(locks.Static)
|
||||
for _, lock := range locks.All {
|
||||
origin := "remote"
|
||||
if _, ok := staticSet[lock.Source]; ok {
|
||||
origin = "pipeline"
|
||||
}
|
||||
promo := "not-promoted"
|
||||
if _, ok := promoted[lock.Source]; ok {
|
||||
promo = "promoted"
|
||||
}
|
||||
reason := strings.TrimSpace(lock.Reason)
|
||||
if reason == "" {
|
||||
reason = "(no reason)"
|
||||
}
|
||||
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func lockMapValues(in map[string]config.ArchiveLockRule) []config.ArchiveLockRule {
|
||||
keys := make([]string, 0, len(in))
|
||||
for key := range in {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]config.ArchiveLockRule, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
item := in[key]
|
||||
item.Source = key
|
||||
item.Reason = strings.TrimSpace(item.Reason)
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
589
internal/app/operator_helpers_test.go
Normal file
589
internal/app/operator_helpers_test.go
Normal file
@@ -0,0 +1,589 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestExecuteSessionInitRemoteWritesCanonicalSessionConfig(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"session", "init",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session-id", "2026-06-07",
|
||||
"--title", "The Black Cabin",
|
||||
"--remote",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
key := artifacts.S3SessionConfigKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-06-07"))
|
||||
obj, ok := fake.Objects[key]
|
||||
if !ok {
|
||||
t.Fatalf("remote session key %q not uploaded; objects=%v", key, fake.Objects)
|
||||
}
|
||||
if !strings.Contains(string(obj.Data), `session_id: "2026-06-07"`) || !strings.Contains(string(obj.Data), "prefix: audio/") {
|
||||
t.Fatalf("remote session data = %q", string(obj.Data))
|
||||
}
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
accessKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write session: %v", err)
|
||||
}
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
audioKey := artifacts.S3PromotedArtifactKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "test-key-id" || os.Getenv(secretKeyEnv) != "test-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "validate", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "OK audio") {
|
||||
t.Fatalf("stdout = %q, want OK audio", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"locks", "add",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--reason", "manual edit",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||
obj, ok := fake.Objects[key]
|
||||
if !ok {
|
||||
t.Fatalf("remote locks key %q not uploaded", key)
|
||||
}
|
||||
if !strings.Contains(string(obj.Data), "source: narratio.transcript.trimmed") || !strings.Contains(string(obj.Data), "reason: manual edit") {
|
||||
t.Fatalf("lock store data = %q", string(obj.Data))
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"locks",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("locks list exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "- narratio.transcript.trimmed origin=remote") {
|
||||
t.Fatalf("stdout = %q, want remote lock", stdout.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"locks", "remove",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
||||
}
|
||||
if len(store.Locks) != 0 {
|
||||
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
||||
}
|
||||
if storeInitCalls != 3 {
|
||||
t.Fatalf("object store init calls = %d, want 3", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"locks", "add",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--reason", "first",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("initial locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"locks", "add",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--reason", "second",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("duplicate locks add exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "pass --force to update") {
|
||||
t.Fatalf("stderr = %q, want force guidance", stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"locks", "add",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--reason", "second",
|
||||
"--force",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("forced locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||
if !strings.Contains(string(fake.Objects[key].Data), "reason: second") {
|
||||
t.Fatalf("lock store data = %q, want updated reason", string(fake.Objects[key].Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocksRequireSessionID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{"list", []string{"locks"}, "locks: --session-id is required"},
|
||||
{"add", []string{"locks", "add", "narratio.transcript.trimmed"}, "locks add: --session-id is required"},
|
||||
{"remove", []string{"locks", "remove", "narratio.transcript.trimmed"}, "locks remove: --session-id is required"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tt.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tt.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocksCannotModifyStaticLocks(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addStaticArchiveLockToPipelineConfig(t, pipelinePath, "narratio.transcript.trimmed")
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"locks", "add",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("locks add static lock exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "locked by pipeline config") {
|
||||
t.Fatalf("stderr = %q, want static lock error", stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"locks", "remove",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"narratio.transcript.trimmed",
|
||||
}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("locks remove static lock exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "locked by pipeline config") {
|
||||
t.Fatalf("stderr = %q, want static lock error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteTopLevelLockAndUnlockAreRemoved(t *testing.T) {
|
||||
tests := []string{"lock", "unlock"}
|
||||
for _, cmd := range tests {
|
||||
t.Run(cmd, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{cmd, "narratio.transcript.trimmed"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown command: "`+cmd+`"`) {
|
||||
t.Fatalf("stderr = %q, want unknown command", stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||
promote_artifacts:
|
||||
- source: narratio.transcript.trimmed
|
||||
dest: transcripts/trimmed.json
|
||||
required: true
|
||||
`)
|
||||
fake := &storage.FakeBackend{}
|
||||
trimmedKey := artifacts.S3PromotedArtifactKey(
|
||||
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||
"transcripts/trimmed.json",
|
||||
)
|
||||
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"artifacts", "list",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--remote",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio.transcript.trimmed remote=promoted") {
|
||||
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||
promote_artifacts:
|
||||
- source: narratio.transcript.full
|
||||
dest: transcripts/full.json
|
||||
required: true
|
||||
- source: narratio.bounds.session
|
||||
dest: transcripts/bounds.json
|
||||
required: true
|
||||
`)
|
||||
fake := &storage.FakeBackend{}
|
||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
|
||||
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"artifacts", "list",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--remote",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
for _, unwanted := range []string{
|
||||
"narratio.transcript.full remote=missing",
|
||||
"narratio.bounds.session remote=missing",
|
||||
} {
|
||||
if strings.Contains(out, unwanted) {
|
||||
t.Fatalf("stdout = %q, did not want catalog remote marker %q", out, unwanted)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
|
||||
"narratio.bounds.session dest=transcripts/bounds.json remote=promoted",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("stdout = %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||
promote_artifacts:
|
||||
- source: narratio.transcript.trimmed
|
||||
dest: transcripts/trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.full
|
||||
dest: transcripts/full.json
|
||||
required: true
|
||||
`)
|
||||
fake := &storage.FakeBackend{}
|
||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
||||
trimmedKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
|
||||
fullKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json")
|
||||
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
|
||||
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
|
||||
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
|
||||
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
|
||||
fake.SeedObject(storage.FakeObject{Key: fullKey, Data: []byte(`{"segments":[]}`)})
|
||||
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"status",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
for _, want := range []string{
|
||||
"Remote outputs:",
|
||||
"Built-in:",
|
||||
"Configured:",
|
||||
"Previous-session:",
|
||||
"Promoted:",
|
||||
"narratio.transcript.trimmed locked",
|
||||
"narratio.transcript.trimmed locked remote=promoted",
|
||||
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("stdout = %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "narratio.transcript.merged remote=missing") {
|
||||
t.Fatalf("stdout = %q, did not want catalog remote marker", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||
promote_artifacts:
|
||||
- source: narratio.transcript.trimmed
|
||||
dest: transcripts/trimmed.json
|
||||
required: true
|
||||
`)
|
||||
fake := &storage.FakeBackend{ExistsErr: fmt.Errorf("exists failed")}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"status",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
if !strings.Contains(out, "Remote archive: missing or unavailable:") {
|
||||
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
|
||||
}
|
||||
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.trimmed remote=error") {
|
||||
t.Fatalf("stdout = %q, want remote output error state", out)
|
||||
}
|
||||
if !strings.Contains(out, "Archive locks: error:") {
|
||||
t.Fatalf("stdout = %q, want archive locks error", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArchiveLoadsRemoteLocks(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidArchiveConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
lockKey := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
|
||||
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||
// The archive stage only checks the manifest statuses and source files.
|
||||
_ = stageName
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "trimmed.json"), `{"segments":[]}`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "archive"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
promotedKey := artifacts.S3PromotedArtifactKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/trimmed.json")
|
||||
if _, ok := fake.Objects[promotedKey]; ok {
|
||||
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
|
||||
}
|
||||
}
|
||||
|
||||
func addArchivePromotionsToPipeline(t *testing.T, pipelinePath, archiveYAML string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline: %v", err)
|
||||
}
|
||||
updated := strings.Replace(string(data), " upload_run: false\n", " upload_run: false\n"+archiveYAML, 1)
|
||||
if updated == string(data) {
|
||||
t.Fatalf("pipeline %q did not contain archive upload_run marker", pipelinePath)
|
||||
}
|
||||
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidArchiveConfigFiles(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||
t.Helper()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline: %v", err)
|
||||
}
|
||||
updated := strings.Replace(string(data), "upload_run: false", "upload_run: true", 1)
|
||||
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", nowUTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.RunID = "20260521T160000Z-test"
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||
}
|
||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err := store.Save(ctx, path, m); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
runManifestPath := artifacts.SessionRunManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, m.RunID)
|
||||
if err := os.MkdirAll(filepath.Dir(runManifestPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir run manifest: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(runManifestPath, []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatalf("write run manifest: %v", err)
|
||||
}
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func addStaticArchiveLockToPipelineConfig(t *testing.T, pipelinePath, source string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline: %v", err)
|
||||
}
|
||||
updated := strings.Replace(
|
||||
string(data),
|
||||
"archive:\n enabled: true\n upload_run: false\n",
|
||||
"archive:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
|
||||
1,
|
||||
)
|
||||
if updated == string(data) {
|
||||
t.Fatalf("archive section not found in pipeline config")
|
||||
}
|
||||
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -20,11 +20,13 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
var force bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
@@ -36,16 +38,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("plan: unexpected positional arguments")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
|
||||
@@ -15,10 +15,10 @@ import (
|
||||
|
||||
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var out bytes.Buffer
|
||||
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
||||
args := []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}
|
||||
|
||||
if err := Plan(context.Background(), args, &out); err != nil {
|
||||
t.Fatalf("first Plan() error = %v", err)
|
||||
@@ -62,7 +62,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
|
||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
|
||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out); err != nil {
|
||||
t.Fatalf("Plan() error = %v", err)
|
||||
}
|
||||
got := out.String()
|
||||
@@ -93,6 +93,7 @@ func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
@@ -128,7 +129,7 @@ inputs:
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
|
||||
@@ -153,53 +153,70 @@ func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool,
|
||||
return true, ""
|
||||
}
|
||||
|
||||
type scopedDir struct {
|
||||
RootAbs string
|
||||
TargetAbs string
|
||||
Exists bool
|
||||
}
|
||||
|
||||
func removeRunScopedDir(root, target, policy string) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !dir.Exists {
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
if cleanRoot == "" {
|
||||
return fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
if cleanTarget == "" {
|
||||
return fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||
}
|
||||
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
targetAbs, err := filepath.Abs(cleanTarget)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||
}
|
||||
if rel == "." {
|
||||
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||
}
|
||||
|
||||
info, err := os.Lstat(targetAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||
}
|
||||
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||
}
|
||||
if err := os.RemoveAll(targetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
|
||||
}
|
||||
return nil
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
|
||||
157
internal/app/remote_locks.go
Normal file
157
internal/app/remote_locks.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type effectiveLocks struct {
|
||||
Static []config.ArchiveLockRule
|
||||
Remote []config.ArchiveLockRule
|
||||
All []config.ArchiveLockRule
|
||||
Key string
|
||||
}
|
||||
|
||||
func remoteLocksKey(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return "", fmt.Errorf("resolved config is required")
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil {
|
||||
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(
|
||||
cfg.Pipeline.Storage.S3.RootPrefix,
|
||||
cfg.Session.Campaign,
|
||||
cfg.Session.SessionID,
|
||||
)
|
||||
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||
}
|
||||
|
||||
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.ArchiveLockStore, string, error) {
|
||||
key, err := remoteLocksKey(cfg)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
exists, err := store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
||||
}
|
||||
if !exists {
|
||||
return &config.ArchiveLockStore{}, key, nil
|
||||
}
|
||||
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp) }()
|
||||
data, err := os.ReadFile(tmp)
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||
}
|
||||
lockStore, err := config.LoadArchiveLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
||||
if err != nil {
|
||||
return nil, key, err
|
||||
}
|
||||
return lockStore, key, nil
|
||||
}
|
||||
|
||||
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
||||
staticLocks := staticArchiveLocks(cfg)
|
||||
if store == nil {
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
All: append([]config.ArchiveLockRule(nil), staticLocks...),
|
||||
}, nil
|
||||
}
|
||||
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteLocks := append([]config.ArchiveLockRule(nil), lockStore.Locks...)
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
Remote: remoteLocks,
|
||||
All: config.MergeArchiveLockRules(staticLocks, remoteLocks),
|
||||
Key: key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func staticArchiveLocks(cfg *config.Config) []config.ArchiveLockRule {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
|
||||
return nil
|
||||
}
|
||||
return append([]config.ArchiveLockRule(nil), cfg.Pipeline.Archive.Locks...)
|
||||
}
|
||||
|
||||
func applyEffectiveLocks(cfg *config.Config, locks []config.ArchiveLockRule) {
|
||||
if cfg == nil || cfg.Pipeline == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Pipeline.Archive == nil {
|
||||
cfg.Pipeline.Archive = &config.ArchiveConfig{}
|
||||
}
|
||||
cfg.Pipeline.Archive.Locks = append([]config.ArchiveLockRule(nil), locks...)
|
||||
}
|
||||
|
||||
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.ArchiveLockStore) error {
|
||||
data, err := config.MarshalArchiveLockStore(lockStore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create lock store temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("write lock store temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close lock store temp file: %w", err)
|
||||
}
|
||||
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockSourceSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||
for _, lock := range locks {
|
||||
source := strings.TrimSpace(lock.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
lock.Source = source
|
||||
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||
out[source] = lock
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeLocalFile(path string, data []byte, force bool) error {
|
||||
cleaned := filepath.Clean(strings.TrimSpace(path))
|
||||
if cleaned == "" || cleaned == "." {
|
||||
return fmt.Errorf("output path is required")
|
||||
}
|
||||
if !force {
|
||||
if _, err := os.Stat(cleaned); err == nil {
|
||||
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
|
||||
return fmt.Errorf("create output directory: %w", err)
|
||||
}
|
||||
return os.WriteFile(cleaned, data, 0o644)
|
||||
}
|
||||
252
internal/app/remote_session_test.go
Normal file
252
internal/app/remote_session_test.go
Normal file
@@ -0,0 +1,252 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio plan: workdir prepared") {
|
||||
t.Fatalf("stdout = %q, want plan output", stdout.String())
|
||||
}
|
||||
if _, ok := fake.Objects[remoteKey]; !ok {
|
||||
t.Fatalf("remote session key %q was not seeded", remoteKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
accessKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
|
||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
|
||||
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), missingSessionPath) {
|
||||
t.Fatalf("stderr = %q, want local searched path", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "remote session loading requires --session-id") {
|
||||
t.Fatalf("stderr = %q, want session-id guidance", stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("storage unavailable")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
|
||||
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "strict decode failed") {
|
||||
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
|
||||
t.Helper()
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if storeInitCalls != nil {
|
||||
(*storeInitCalls)++
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
}
|
||||
|
||||
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
|
||||
t.Helper()
|
||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
fake.SeedObject(storage.FakeObject{
|
||||
Key: remoteKey,
|
||||
Data: []byte(content),
|
||||
ETag: "remote-session-etag",
|
||||
})
|
||||
return remoteKey
|
||||
}
|
||||
|
||||
func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv string) {
|
||||
t.Helper()
|
||||
pipelineData, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline: %v", err)
|
||||
}
|
||||
pipelineYAML := strings.Replace(
|
||||
string(pipelineData),
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
|
||||
1,
|
||||
)
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.SetOutput(out)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
@@ -33,6 +34,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
var force bool
|
||||
var includeAudio bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
@@ -40,7 +42,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
||||
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
||||
fs.Usage = func() {
|
||||
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
||||
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--campaign <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
||||
_, _ = fmt.Fprintln(out)
|
||||
_, _ = fmt.Fprintln(out, "Flags:")
|
||||
fs.PrintDefaults()
|
||||
@@ -55,16 +57,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("restore: unexpected positional arguments")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
@@ -74,11 +67,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
|
||||
objectStore, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||
objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/audio"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
@@ -92,6 +93,10 @@ func executeRestoreDownloadAction(
|
||||
return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath)
|
||||
}
|
||||
|
||||
if restoreActionIsAudio(action) {
|
||||
return executeRestoreAudioAction(ctx, cfg, safeLocalPath, action, store)
|
||||
}
|
||||
|
||||
tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download to temp file: %w", err)
|
||||
@@ -120,6 +125,38 @@ func executeRestoreDownloadAction(
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeRestoreAudioAction(
|
||||
ctx context.Context,
|
||||
cfg *config.Config,
|
||||
safeLocalPath string,
|
||||
action RestoreAction,
|
||||
store storage.ObjectStore,
|
||||
) error {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || cfg.Session == nil {
|
||||
return fmt.Errorf("resolved s3 config and session are required")
|
||||
}
|
||||
spoolDir := artifacts.SessionSpoolRestoreAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
spoolPath := filepath.Join(spoolDir, filepath.Base(safeLocalPath))
|
||||
cacheEnabled := cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio
|
||||
_, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||
Store: store,
|
||||
Object: storage.ObjectInfo{
|
||||
Key: action.RemoteKey,
|
||||
Size: action.Size,
|
||||
ETag: action.ETag,
|
||||
},
|
||||
Bucket: strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket),
|
||||
CacheRoot: strings.TrimSpace(cfg.Pipeline.Cache.Root),
|
||||
CacheEnabled: cacheEnabled,
|
||||
SpoolPath: spoolPath,
|
||||
DestPath: safeLocalPath,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("materialize audio: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) {
|
||||
if strings.TrimSpace(destPath) == "" {
|
||||
return "", fmt.Errorf("destination path is required")
|
||||
|
||||
@@ -18,10 +18,10 @@ import (
|
||||
|
||||
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
|
||||
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
|
||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||
@@ -32,7 +32,7 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -64,17 +64,17 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||
|
||||
restoreWithStoreAndRealPhases(t, fake)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -87,12 +87,50 @@ func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||
func TestExecuteRestoreIncludeAudioUsesCacheAfterWorkspaceDeletion(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
audioKey := sessionPrefix + "audio/alice.flac"
|
||||
seedRestoreObject(fake, audioKey, []byte("remote-audio"))
|
||||
|
||||
restoreWithStoreAndRealPhases(t, fake)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("first restore exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if got := fakeDownloadCount(fake, audioKey); got != 1 {
|
||||
t.Fatalf("audio downloads after first restore = %d, want 1", got)
|
||||
}
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
|
||||
|
||||
if err := os.RemoveAll(sessionRoot); err != nil {
|
||||
t.Fatalf("remove session root: %v", err)
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("second restore exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if got := fakeDownloadCount(fake, audioKey); got != 1 {
|
||||
t.Fatalf("audio downloads after cached restore = %d, want still 1", got)
|
||||
}
|
||||
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
|
||||
}
|
||||
|
||||
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
||||
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
||||
|
||||
@@ -100,7 +138,7 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -116,10 +154,10 @@ func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
@@ -129,7 +167,7 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -148,10 +186,10 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
@@ -161,7 +199,7 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -174,10 +212,10 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
|
||||
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
@@ -187,7 +225,7 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -196,10 +234,10 @@ func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||
|
||||
store := artifacts.NewLocalStore(workspaceRoot)
|
||||
@@ -213,7 +251,7 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -229,10 +267,10 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
base := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||
|
||||
toggled := &stagedManifestDownloadStore{
|
||||
@@ -260,7 +298,7 @@ func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -344,9 +382,9 @@ func restoreWithStoreAndRealPhases(t *testing.T, objectStore storage.ObjectStore
|
||||
executeRestorePlanFn = executeRestorePlan
|
||||
}
|
||||
|
||||
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, sessionPath string) (*config.Config, string, string, string) {
|
||||
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, campaignPath, sessionPath string) (*config.Config, string, string, string) {
|
||||
t.Helper()
|
||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, sessionPath, config.SessionLoadOptions{})
|
||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
@@ -374,6 +412,16 @@ func mustReadEquals(t *testing.T, path, want string) {
|
||||
}
|
||||
}
|
||||
|
||||
func fakeDownloadCount(fake *storage.FakeBackend, key string) int {
|
||||
count := 0
|
||||
for _, call := range fake.Downloads {
|
||||
if call.Key == key {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
type stagedManifestDownloadStore struct {
|
||||
delegate *storage.FakeBackend
|
||||
manifestKey string
|
||||
|
||||
@@ -257,6 +257,35 @@ func classifyRestoreAction(
|
||||
return action, nil
|
||||
}
|
||||
|
||||
if restoreRelativePathIsAudio(localRelPath) {
|
||||
if object.Size > 0 {
|
||||
if info.Size() == object.Size {
|
||||
action.Kind = RestoreActionSkipSame
|
||||
action.SameLocal = true
|
||||
action.Reason = "local audio size matches remote content"
|
||||
return action, nil
|
||||
}
|
||||
if force {
|
||||
action.Kind = RestoreActionDownload
|
||||
action.Reason = "local audio differs (size mismatch); overwrite with --force"
|
||||
return action, nil
|
||||
}
|
||||
action.Kind = RestoreActionConflict
|
||||
action.Conflict = true
|
||||
action.Reason = "local audio differs (size mismatch)"
|
||||
return action, nil
|
||||
}
|
||||
if force {
|
||||
action.Kind = RestoreActionDownload
|
||||
action.Reason = "local audio exists; remote size unavailable; overwrite with --force"
|
||||
return action, nil
|
||||
}
|
||||
action.Kind = RestoreActionConflict
|
||||
action.Conflict = true
|
||||
action.Reason = "local audio exists; remote size unavailable"
|
||||
return action, nil
|
||||
}
|
||||
|
||||
if object.Size > 0 && info.Size() != object.Size {
|
||||
if force {
|
||||
action.Kind = RestoreActionDownload
|
||||
@@ -303,6 +332,15 @@ func classifyRestoreAction(
|
||||
return action, nil
|
||||
}
|
||||
|
||||
func restoreActionIsAudio(action RestoreAction) bool {
|
||||
return restoreRelativePathIsAudio(action.LocalRelativePath)
|
||||
}
|
||||
|
||||
func restoreRelativePathIsAudio(rel string) bool {
|
||||
cleanRel := path.Clean(strings.TrimSpace(rel))
|
||||
return cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")
|
||||
}
|
||||
|
||||
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
|
||||
if out == nil {
|
||||
return fmt.Errorf("output writer is required")
|
||||
|
||||
@@ -58,6 +58,33 @@ func TestRestorePlanIncludeAudio(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestorePlanExistingAudioUsesSizeWithoutRemoteChecksumDownload(t *testing.T) {
|
||||
cfg := restorePlanConfig(t)
|
||||
current := restorePlanCurrentState(t, cfg)
|
||||
store := &storage.FakeBackend{}
|
||||
|
||||
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
|
||||
seedRestoreObject(store, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
mustWriteTestFile(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "local")
|
||||
|
||||
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{IncludeAudio: true})
|
||||
if err != nil {
|
||||
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||
}
|
||||
if len(store.Downloads) != 0 {
|
||||
t.Fatalf("downloads = %d, want no remote checksum download for audio", len(store.Downloads))
|
||||
}
|
||||
actionByRel := map[string]RestoreAction{}
|
||||
for _, action := range plan.Actions {
|
||||
actionByRel[action.LocalRelativePath] = action
|
||||
}
|
||||
audioAction := actionByRel["audio/alice.flac"]
|
||||
if audioAction.Kind != RestoreActionSkipSame {
|
||||
t.Fatalf("audio action kind = %q, want %q", audioAction.Kind, RestoreActionSkipSame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
|
||||
cfg := restorePlanConfig(t)
|
||||
current := restorePlanCurrentState(t, cfg)
|
||||
|
||||
@@ -32,6 +32,9 @@ func TestExecuteRestoreHelp(t *testing.T) {
|
||||
if !strings.Contains(out, "--include-audio") {
|
||||
t.Fatalf("stdout = %q, want --include-audio flag", out)
|
||||
}
|
||||
if !strings.Contains(out, "--campaign") {
|
||||
t.Fatalf("stdout = %q, want --campaign flag", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||
@@ -70,7 +73,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||
}
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -78,6 +81,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||
[]string{
|
||||
"restore",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--dry-run",
|
||||
@@ -116,11 +120,11 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||
|
||||
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -138,11 +142,11 @@ func TestExecuteRestoreFailsWhenStorageBackendNotConfigured(t *testing.T) {
|
||||
})
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -166,11 +170,11 @@ func TestExecuteRestoreDiscoveryErrorSurfaced(t *testing.T) {
|
||||
}
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -208,7 +212,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
restoreEnv(secretKeyEnv)
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
||||
@@ -261,6 +265,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
[]string{
|
||||
"restore",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--dry-run",
|
||||
@@ -307,10 +312,10 @@ func TestExecuteRestoreNonDryRunConflictFailsBeforeNYI(t *testing.T) {
|
||||
}
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -359,10 +364,10 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
||||
}
|
||||
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -374,11 +379,12 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string) {
|
||||
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, dir)
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
@@ -405,5 +411,5 @@ inputs:
|
||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, sessionPath
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
@@ -19,10 +19,10 @@ import (
|
||||
|
||||
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
|
||||
@@ -48,6 +48,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||
[]string{
|
||||
"restore",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", cfg.Session.SessionID,
|
||||
},
|
||||
@@ -86,6 +87,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||
[]string{
|
||||
"run-stage",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", cfg.Session.SessionID,
|
||||
"--force",
|
||||
@@ -157,7 +159,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||
|
||||
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
@@ -176,7 +178,7 @@ scriptorium:
|
||||
`)
|
||||
|
||||
fakeStore := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, sessionPath)
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
|
||||
@@ -191,6 +193,7 @@ scriptorium:
|
||||
[]string{
|
||||
"restore",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", cfg.Session.SessionID,
|
||||
},
|
||||
@@ -235,6 +238,7 @@ scriptorium:
|
||||
[]string{
|
||||
"run-stage",
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", cfg.Session.SessionID,
|
||||
"--force",
|
||||
|
||||
@@ -17,12 +17,14 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
var force bool
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
@@ -35,16 +37,7 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("resume: unexpected positional arguments")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume: %w", err)
|
||||
}
|
||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -32,7 +32,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Resume() error = %v", err)
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||
|
||||
func TestResumeNoRemainingStages(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -64,7 +64,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Resume() error = %v", err)
|
||||
}
|
||||
@@ -80,7 +80,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
||||
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -93,7 +93,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force"}, &out)
|
||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Resume() error = %v", err)
|
||||
}
|
||||
@@ -104,14 +104,14 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
||||
|
||||
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage() error = %v", err)
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||
|
||||
func TestRunStageSkipAndForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||
@@ -148,7 +148,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage() error = %v", err)
|
||||
}
|
||||
@@ -157,7 +157,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(force) error = %v", err)
|
||||
}
|
||||
@@ -168,7 +168,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
||||
|
||||
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||
@@ -184,7 +184,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(force) error = %v", err)
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||
err = Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Resume() error = %v", err)
|
||||
}
|
||||
@@ -214,13 +214,13 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
||||
|
||||
func TestRunStageTrimExecutes(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "trim"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "trim"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(trim) error = %v", err)
|
||||
}
|
||||
@@ -243,13 +243,13 @@ func TestRunStageTrimExecutes(t *testing.T) {
|
||||
|
||||
func TestRunStageNormalizeExecutes(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "normalize"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(normalize) error = %v", err)
|
||||
}
|
||||
|
||||
@@ -15,12 +15,14 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
var force bool
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
@@ -33,16 +35,7 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("run: unexpected positional arguments")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run: %w", err)
|
||||
}
|
||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
|
||||
@@ -15,12 +15,14 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
var force bool
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
@@ -33,54 +35,34 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
||||
if fs.NArg() != 1 {
|
||||
return fmt.Errorf("run-stage: expected exactly one stage name")
|
||||
}
|
||||
stageName := fs.Arg(0)
|
||||
normalizedArtifacts, err := selectedArtifacts.Normalize()
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
|
||||
}
|
||||
stageName := fs.Arg(0)
|
||||
if len(normalizedArtifacts) > 0 && stageName != "analyze" {
|
||||
return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"")
|
||||
}
|
||||
stages, err := BuildSingleStagePlan(stageName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||
CommandName: "run-stage",
|
||||
StageName: stageName,
|
||||
PipelinePath: pipelinePath,
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
if err := validateSelectedAnalyzeArtifacts(cfg, normalizedArtifacts); err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
}
|
||||
|
||||
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
|
||||
Force: force,
|
||||
SelectedArtifacts: normalizedArtifacts,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("run-stage: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n",
|
||||
stages[0].Name(),
|
||||
stageName,
|
||||
len(summary.Executed),
|
||||
len(summary.Skipped),
|
||||
force,
|
||||
@@ -88,3 +70,99 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// Analyze force-runs the analyze stage.
|
||||
func Analyze(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("analyze", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var campaignPath string
|
||||
var sessionPath string
|
||||
var sessionID string
|
||||
var previousSessionID string
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("analyze: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("analyze: unexpected positional arguments")
|
||||
}
|
||||
normalizedArtifacts, err := selectedArtifacts.Normalize()
|
||||
if err != nil {
|
||||
return fmt.Errorf("analyze: invalid --artifacts: %w", err)
|
||||
}
|
||||
|
||||
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||
CommandName: "analyze",
|
||||
StageName: "analyze",
|
||||
PipelinePath: pipelinePath,
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
Force: true,
|
||||
SelectedArtifacts: normalizedArtifacts,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = fmt.Fprintf(
|
||||
out,
|
||||
"narratio analyze: executed=%d skipped=%d force=true; manifest=%s\n",
|
||||
len(summary.Executed),
|
||||
len(summary.Skipped),
|
||||
summary.ManifestPath,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
type singleStageCommand struct {
|
||||
CommandName string
|
||||
StageName string
|
||||
PipelinePath string
|
||||
CampaignPath string
|
||||
SessionPath string
|
||||
SessionID string
|
||||
PreviousSessionID string
|
||||
Force bool
|
||||
SelectedArtifacts []string
|
||||
}
|
||||
|
||||
func runSingleStageCommand(ctx context.Context, req singleStageCommand) (*RunSummary, error) {
|
||||
stages, err := BuildSingleStagePlan(req.StageName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
|
||||
cfg, err := loadCommandConfig(ctx, req.PipelinePath, req.CampaignPath, req.SessionPath, config.SessionLoadOptions{
|
||||
SessionID: req.SessionID,
|
||||
PreviousSessionID: req.PreviousSessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
if err := validateSelectedAnalyzeArtifacts(cfg, req.SelectedArtifacts); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
|
||||
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
|
||||
Force: req.Force,
|
||||
SelectedArtifacts: req.SelectedArtifacts,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
@@ -87,12 +87,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
env.Storage = &storage.NoopBackend{}
|
||||
}
|
||||
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
|
||||
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
|
||||
objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
env.ObjectStore = objectStore
|
||||
}
|
||||
if needsRemoteLocksForRun(env.Config, stages) {
|
||||
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load remote archive locks: %w", err)
|
||||
}
|
||||
applyEffectiveLocks(env.Config, locks.All)
|
||||
}
|
||||
if env.Notifier == nil {
|
||||
env.Notifier = ¬ify.NoopSender{}
|
||||
}
|
||||
@@ -576,6 +583,32 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return false
|
||||
}
|
||||
archiveRequested := false
|
||||
for _, s := range stages {
|
||||
if s != nil && s.Name() == "archive" {
|
||||
archiveRequested = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !archiveRequested {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive == nil {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
||||
return false
|
||||
}
|
||||
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
||||
return false
|
||||
}
|
||||
return cfg.Pipeline.Storage.S3 != nil
|
||||
}
|
||||
|
||||
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||
return nil
|
||||
|
||||
@@ -994,10 +994,12 @@ func testConfig(t *testing.T) *config.Config {
|
||||
workspace := t.TempDir()
|
||||
cfgDir := t.TempDir()
|
||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||
|
||||
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||
mustWriteFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n")
|
||||
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||
@@ -1005,8 +1007,27 @@ func testConfig(t *testing.T) *config.Config {
|
||||
|
||||
return &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
||||
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||
PipelinePath: pipelinePath,
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
StableInputs: config.ResolvedStableInputs{
|
||||
SpeakersFile: config.ResolvedInputFile{
|
||||
Path: "./speakers.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
AutocorrectFile: config.ResolvedInputFile{
|
||||
Path: "./autocorrect.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
GlossaryFile: config.ResolvedInputFile{
|
||||
Path: "./glossary.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
},
|
||||
Session: &config.SessionConfig{
|
||||
SessionID: "2026-05-03",
|
||||
Campaign: "sample-campaign",
|
||||
@@ -1023,6 +1044,7 @@ func testConfig(t *testing.T) *config.Config {
|
||||
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + t.TempDir() + `
|
||||
@@ -1032,6 +1054,12 @@ analyzer:
|
||||
timeout: 20m
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
campaignYAML := `campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
@@ -1042,6 +1070,7 @@ inputs:
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
mustWriteFile(t, pipelinePath, pipelineYAML)
|
||||
mustWriteFile(t, campaignPath, campaignYAML)
|
||||
mustWriteFile(t, sessionPath, sessionYAML)
|
||||
|
||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
||||
|
||||
@@ -4,14 +4,15 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
sessionTemplate := `session_id: "{{ session_id }}"
|
||||
previous_session_id: "{{ previous_session_id }}"
|
||||
@@ -26,19 +27,14 @@ inputs:
|
||||
t.Fatalf("write session template: %v", err)
|
||||
}
|
||||
|
||||
cwd := filepath.Dir(sessionPath)
|
||||
originalWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Getwd(): %v", err)
|
||||
}
|
||||
if err := os.Chdir(cwd); err != nil {
|
||||
t.Fatalf("Chdir(%q): %v", cwd, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{sessionPath}
|
||||
t.Cleanup(func() { config.DefaultSessionConfigSearchPaths = origSessionDefaults })
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := Plan(context.Background(), []string{
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session-id", "2026-04-04",
|
||||
"--previous-session-id", "2026-03-28",
|
||||
}, &out); err != nil {
|
||||
@@ -51,10 +47,10 @@ inputs:
|
||||
|
||||
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -65,7 +61,7 @@ func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||
|
||||
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
previous_session_id: 2026-04-26
|
||||
@@ -83,6 +79,7 @@ inputs:
|
||||
var out bytes.Buffer
|
||||
err := Plan(context.Background(), []string{
|
||||
"--config", pipelinePath,
|
||||
"--campaign", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--previous-session-id", "2026-04-25",
|
||||
@@ -97,10 +94,10 @@ inputs:
|
||||
|
||||
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage() error = %v", err)
|
||||
}
|
||||
@@ -110,7 +107,7 @@ func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
|
||||
_, err := resolveSessionConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
|
||||
@@ -19,6 +19,22 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
||||
return explicit, nil
|
||||
}
|
||||
|
||||
resolved, err := discoverSessionConfigPathWithCandidates(candidates)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resolved.Path != "" {
|
||||
return resolved.Path, nil
|
||||
}
|
||||
return "", missingSessionConfigError(resolved.Searched, "")
|
||||
}
|
||||
|
||||
type sessionConfigDiscovery struct {
|
||||
Path string
|
||||
Searched []string
|
||||
}
|
||||
|
||||
func discoverSessionConfigPathWithCandidates(candidates []string) (sessionConfigDiscovery, error) {
|
||||
ordered := make([]string, 0, len(candidates))
|
||||
for _, raw := range candidates {
|
||||
path := strings.TrimSpace(raw)
|
||||
@@ -31,19 +47,32 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
||||
if info.IsDir() {
|
||||
continue
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
return sessionConfigDiscovery{Path: filepath.Clean(path), Searched: ordered}, nil
|
||||
}
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return "", fmt.Errorf("check default session config %q: %w", path, err)
|
||||
return sessionConfigDiscovery{}, fmt.Errorf("check default session config %q: %w", path, err)
|
||||
}
|
||||
|
||||
return sessionConfigDiscovery{Searched: ordered}, nil
|
||||
}
|
||||
|
||||
func missingSessionConfigError(searched []string, remoteDetail string) error {
|
||||
ordered := append([]string(nil), searched...)
|
||||
if len(ordered) == 0 {
|
||||
return "", fmt.Errorf("no session config path provided and no default locations configured")
|
||||
if strings.TrimSpace(remoteDetail) != "" {
|
||||
return fmt.Errorf("no session config path provided and no default locations configured; %s", remoteDetail)
|
||||
}
|
||||
return fmt.Errorf("no session config path provided and no default locations configured")
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
|
||||
msg := fmt.Sprintf(
|
||||
"no session config path provided and no default session config found; searched: %s",
|
||||
strings.Join(ordered, ", "),
|
||||
)
|
||||
if strings.TrimSpace(remoteDetail) != "" {
|
||||
msg += "; " + strings.TrimSpace(remoteDetail)
|
||||
}
|
||||
msg += "; pass --session to use an explicit path"
|
||||
return fmt.Errorf("%s", msg)
|
||||
}
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
// Status reads and prints stage statuses from an existing manifest.
|
||||
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var manifestPath string
|
||||
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("status: invalid flags: %w", err)
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("status: unexpected positional arguments")
|
||||
}
|
||||
if manifestPath == "" {
|
||||
return fmt.Errorf("status: --manifest is required")
|
||||
}
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m, err := store.Load(ctx, manifestPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(m.Stages) == 0 {
|
||||
_, err := fmt.Fprintln(out, "stages: no stages recorded")
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintln(out, "stages:"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(m.Stages))
|
||||
for name := range m.Stages {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
for _, name := range names {
|
||||
status := m.Stages[name].Status
|
||||
if _, err := fmt.Fprintf(out, "- %s: %s\n", name, status); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -409,6 +410,17 @@ func TestResolvePreviousSessionArtifactWithCatalogFallsBackToPreparedCachePath(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviousSessionCacheCandidatePathsStripsArtifactsPrefix(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||
|
||||
got := previousSessionCacheCandidatePaths(paths, "artifacts/session_recap.md")
|
||||
want := []string{filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("previousSessionCacheCandidatePaths() = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
@@ -64,7 +67,7 @@ func SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID string)
|
||||
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
|
||||
return filepath.Join(
|
||||
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
|
||||
filepath.FromSlash(artifactRelativePath),
|
||||
filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -88,6 +91,60 @@ func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||
}
|
||||
|
||||
// SessionSpoolDir returns the campaign/session scoped local spool root.
|
||||
func SessionSpoolDir(spoolRoot, campaign, sessionID string) string {
|
||||
return filepath.Join(spoolRoot, campaign, sessionID)
|
||||
}
|
||||
|
||||
// SessionSpoolRestoreAudioDir returns the local spool audio path for restore downloads.
|
||||
func SessionSpoolRestoreAudioDir(spoolRoot, campaign, sessionID string) string {
|
||||
return filepath.Join(SessionSpoolDir(spoolRoot, campaign, sessionID), "restore", config.PathAudioDirSegment)
|
||||
}
|
||||
|
||||
// S3AudioCachePath returns the durable local cache path for one S3 audio object.
|
||||
func S3AudioCachePath(cacheRoot, bucket, key string) (string, error) {
|
||||
root := filepath.Clean(strings.TrimSpace(cacheRoot))
|
||||
if root == "." || root == "" {
|
||||
return "", fmt.Errorf("cache root is required")
|
||||
}
|
||||
bucket = strings.Trim(strings.TrimSpace(bucket), "/")
|
||||
if bucket == "" || bucket == "." || bucket == ".." || strings.Contains(bucket, "/") || strings.Contains(bucket, `\`) {
|
||||
return "", fmt.Errorf("bucket is required and must be a single path segment")
|
||||
}
|
||||
rawKey := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
|
||||
if strings.HasPrefix(rawKey, "/") {
|
||||
return "", fmt.Errorf("s3 key must not be absolute")
|
||||
}
|
||||
cleanKey := cleanCacheS3Key(rawKey)
|
||||
if cleanKey == "" {
|
||||
return "", fmt.Errorf("s3 key is required")
|
||||
}
|
||||
if cleanKey == ".." || strings.HasPrefix(cleanKey, "../") || strings.HasPrefix(cleanKey, "/") {
|
||||
return "", fmt.Errorf("s3 key must not escape cache root")
|
||||
}
|
||||
return filepath.Join(root, "s3", bucket, filepath.FromSlash(cleanKey)), nil
|
||||
}
|
||||
|
||||
// S3AudioCacheNamespaceDir returns the durable local cache namespace for all
|
||||
// Narratio S3 audio objects under one bucket/root prefix.
|
||||
func S3AudioCacheNamespaceDir(cacheRoot, bucket, rootPrefix string) (string, error) {
|
||||
sentinelKey := path.Join(cleanS3PathPart(rootPrefix), config.S3CampaignsSegment, ".narratio-cache-sentinel")
|
||||
sentinelPath, err := S3AudioCachePath(cacheRoot, bucket, sentinelKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Dir(sentinelPath), nil
|
||||
}
|
||||
|
||||
func cleanCacheS3Key(key string) string {
|
||||
normalized := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
|
||||
normalized = strings.Trim(normalized, "/")
|
||||
if normalized == "" {
|
||||
return ""
|
||||
}
|
||||
return path.Clean(normalized)
|
||||
}
|
||||
|
||||
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
|
||||
func SessionPreviousDir(paths SessionPaths) string {
|
||||
return paths.PreviousDir
|
||||
@@ -105,7 +162,19 @@ func SessionPreviousArtifactsDir(paths SessionPaths) string {
|
||||
|
||||
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
|
||||
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
|
||||
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(artifactRelativePath))
|
||||
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)))
|
||||
}
|
||||
|
||||
func previousArtifactCacheRelativePath(artifactRelativePath string) string {
|
||||
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimSpace(artifactRelativePath))))
|
||||
if rel == "." {
|
||||
return ""
|
||||
}
|
||||
const artifactsPrefix = "artifacts/"
|
||||
if strings.HasPrefix(rel, artifactsPrefix) && len(rel) > len(artifactsPrefix) {
|
||||
return strings.TrimPrefix(rel, artifactsPrefix)
|
||||
}
|
||||
return rel
|
||||
}
|
||||
|
||||
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
||||
|
||||
@@ -74,6 +74,11 @@ func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
||||
if artifactPath != wantArtifactPath {
|
||||
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
|
||||
}
|
||||
|
||||
archiveRelativeArtifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "artifacts/session_recap.md")
|
||||
if archiveRelativeArtifactPath != wantArtifactPath {
|
||||
t.Fatalf("SessionPreviousArtifactPathForCampaign(archive-relative) = %q, want %q", archiveRelativeArtifactPath, wantArtifactPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
||||
@@ -93,6 +98,12 @@ func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
||||
if got != want {
|
||||
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
got = SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||
want = filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")
|
||||
if got != want {
|
||||
t.Fatalf("SessionPreviousArtifactPath(archive-relative) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionSpoolAudioDir(t *testing.T) {
|
||||
@@ -103,3 +114,66 @@ func TestSessionSpoolAudioDir(t *testing.T) {
|
||||
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionSpoolDir(t *testing.T) {
|
||||
root := "/var/spool/narratio"
|
||||
got := SessionSpoolDir(root, "forsaken", "2026-04-19")
|
||||
want := filepath.Join(root, "forsaken", "2026-04-19")
|
||||
if got != want {
|
||||
t.Fatalf("SessionSpoolDir() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionSpoolRestoreAudioDir(t *testing.T) {
|
||||
root := "/var/spool/narratio"
|
||||
got := SessionSpoolRestoreAudioDir(root, "forsaken", "2026-04-19")
|
||||
want := filepath.Join(root, "forsaken", "2026-04-19", "restore", "audio")
|
||||
if got != want {
|
||||
t.Fatalf("SessionSpoolRestoreAudioDir() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3AudioCachePath(t *testing.T) {
|
||||
got, err := S3AudioCachePath("/var/cache/narratio", "my-dnd-archive", "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns", "forsaken", "sessions", "2026-04-19", "audio", "alice.flac")
|
||||
if got != want {
|
||||
t.Fatalf("S3AudioCachePath() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3AudioCacheNamespaceDir(t *testing.T) {
|
||||
got, err := S3AudioCacheNamespaceDir("/var/cache/narratio", "my-dnd-archive", "dnd")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCacheNamespaceDir() error = %v", err)
|
||||
}
|
||||
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns")
|
||||
if got != want {
|
||||
t.Fatalf("S3AudioCacheNamespaceDir() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3AudioCachePathRejectsUnsafeInputs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
bucket string
|
||||
key string
|
||||
}{
|
||||
{name: "empty root", root: "", bucket: "bucket", key: "audio/a.flac"},
|
||||
{name: "empty bucket", root: "/cache", bucket: "", key: "audio/a.flac"},
|
||||
{name: "bucket slash", root: "/cache", bucket: "bad/bucket", key: "audio/a.flac"},
|
||||
{name: "empty key", root: "/cache", bucket: "bucket", key: ""},
|
||||
{name: "escaping key", root: "/cache", bucket: "bucket", key: "../audio/a.flac"},
|
||||
{name: "absolute key", root: "/cache", bucket: "bucket", key: "/audio/a.flac"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got, err := S3AudioCachePath(tt.root, tt.bucket, tt.key); err == nil {
|
||||
t.Fatalf("S3AudioCachePath() = %q, want error", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,18 @@ func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
|
||||
return ensureS3TrailingSlash(key)
|
||||
}
|
||||
|
||||
// S3SessionConfigKey returns the session config key.
|
||||
// Format: {session_prefix}/session.yml
|
||||
func S3SessionConfigKey(sessionPrefix string) string {
|
||||
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "session.yml")
|
||||
}
|
||||
|
||||
// S3SessionLocksKey returns the mutable session lock store key.
|
||||
// Format: {session_prefix}/locks.yml
|
||||
func S3SessionLocksKey(sessionPrefix string) string {
|
||||
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "locks.yml")
|
||||
}
|
||||
|
||||
// S3CurrentManifestKey returns the current manifest pointer key.
|
||||
// Format: {session_prefix}/current/manifest.json
|
||||
func S3CurrentManifestKey(sessionPrefix string) string {
|
||||
|
||||
@@ -17,6 +17,16 @@ func TestS3KeyConstruction(t *testing.T) {
|
||||
t.Fatalf("audioPrefix = %q", audioPrefix)
|
||||
}
|
||||
|
||||
sessionConfigKey := S3SessionConfigKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
|
||||
if sessionConfigKey != "dnd/campaigns/forsaken/sessions/2026-04-19/session.yml" {
|
||||
t.Fatalf("session config key = %q", sessionConfigKey)
|
||||
}
|
||||
|
||||
locksKey := S3SessionLocksKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
|
||||
if locksKey != "dnd/campaigns/forsaken/sessions/2026-04-19/locks.yml" {
|
||||
t.Fatalf("locks key = %q", locksKey)
|
||||
}
|
||||
|
||||
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
||||
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
||||
if runPrefix != wantRunPrefix {
|
||||
|
||||
223
internal/audio/s3_audio.go
Normal file
223
internal/audio/s3_audio.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
// S3MaterializeRequest describes one S3-backed audio materialization.
|
||||
type S3MaterializeRequest struct {
|
||||
Store storage.ObjectStore
|
||||
Object storage.ObjectInfo
|
||||
Bucket string
|
||||
CacheRoot string
|
||||
CacheEnabled bool
|
||||
SpoolPath string
|
||||
DestPath string
|
||||
}
|
||||
|
||||
// S3MaterializeResult captures local materialization provenance.
|
||||
type S3MaterializeResult struct {
|
||||
Checksum string
|
||||
CachePath string
|
||||
SpoolPath string
|
||||
CacheHit bool
|
||||
Downloaded bool
|
||||
}
|
||||
|
||||
// MaterializeS3Audio installs one S3 audio object into the destination path,
|
||||
// reusing and refreshing the durable local cache when enabled.
|
||||
func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3MaterializeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return S3MaterializeResult{}, err
|
||||
}
|
||||
if req.Store == nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("object store is required")
|
||||
}
|
||||
key := strings.TrimSpace(req.Object.Key)
|
||||
if key == "" {
|
||||
return S3MaterializeResult{}, fmt.Errorf("s3 object key is required")
|
||||
}
|
||||
if strings.TrimSpace(req.DestPath) == "" {
|
||||
return S3MaterializeResult{}, fmt.Errorf("destination path is required")
|
||||
}
|
||||
|
||||
result := S3MaterializeResult{}
|
||||
if req.CacheEnabled {
|
||||
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||
if err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("resolve audio cache path: %w", err)
|
||||
}
|
||||
result.CachePath = cachePath
|
||||
if ok, err := validCachedAudio(cachePath, req.Object.Size); err != nil {
|
||||
return S3MaterializeResult{}, err
|
||||
} else if ok {
|
||||
checksum, err := copyFileAtomicWithChecksum(cachePath, req.DestPath, 0o644)
|
||||
if err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("materialize cached audio %q: %w", cachePath, err)
|
||||
}
|
||||
result.Checksum = checksum
|
||||
result.CacheHit = true
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
spoolPath := strings.TrimSpace(req.SpoolPath)
|
||||
if spoolPath == "" {
|
||||
return S3MaterializeResult{}, fmt.Errorf("spool path is required for s3 audio download")
|
||||
}
|
||||
if err := downloadObjectAtomic(ctx, req.Store, key, spoolPath); err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("download s3 audio object %q: %w", key, err)
|
||||
}
|
||||
if err := validateLocalAudio(spoolPath, req.Object.Size); err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("validate downloaded audio %q: %w", spoolPath, err)
|
||||
}
|
||||
|
||||
checksum, err := copyFileAtomicWithChecksum(spoolPath, req.DestPath, 0o644)
|
||||
if err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("materialize downloaded audio %q: %w", filepath.Base(req.DestPath), err)
|
||||
}
|
||||
result.Checksum = checksum
|
||||
result.SpoolPath = spoolPath
|
||||
result.Downloaded = true
|
||||
|
||||
if result.CachePath != "" {
|
||||
if _, err := copyFileAtomicWithChecksum(spoolPath, result.CachePath, 0o644); err != nil {
|
||||
return S3MaterializeResult{}, fmt.Errorf("populate audio cache %q: %w", result.CachePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validCachedAudio(path string, expectedSize int64) (bool, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("stat cached audio %q: %w", path, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return false, fmt.Errorf("cached audio path is a directory: %q", path)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
if expectedSize > 0 && info.Size() != expectedSize {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func validateLocalAudio(path string, expectedSize int64) error {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat: %w", err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("path is a directory")
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return fmt.Errorf("file is empty")
|
||||
}
|
||||
if expectedSize > 0 && info.Size() != expectedSize {
|
||||
return fmt.Errorf("size %d does not match remote size %d", info.Size(), expectedSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadObjectAtomic(ctx context.Context, store storage.ObjectStore, key, destPath string) error {
|
||||
if strings.TrimSpace(destPath) == "" {
|
||||
return fmt.Errorf("destination path is required")
|
||||
}
|
||||
dir := filepath.Dir(destPath)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create destination directory: %w", err)
|
||||
}
|
||||
base := filepath.Base(destPath)
|
||||
tmp, err := os.CreateTemp(dir, "."+base+".download-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
removeTmp := true
|
||||
defer func() {
|
||||
if removeTmp {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := store.Download(ctx, key, tmpPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(tmpPath, 0o644); err != nil {
|
||||
return fmt.Errorf("set temp file permissions: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, destPath); err != nil {
|
||||
return fmt.Errorf("install downloaded file: %w", err)
|
||||
}
|
||||
removeTmp = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFileAtomicWithChecksum(src, dst string, perm os.FileMode) (string, error) {
|
||||
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
|
||||
return "", fmt.Errorf("source and destination paths are required")
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return "", fmt.Errorf("create destination directory: %w", err)
|
||||
}
|
||||
base := filepath.Base(dst)
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+base+".tmp-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
removeTmp := true
|
||||
defer func() {
|
||||
if removeTmp {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
digest := sha256.New()
|
||||
if _, err := io.Copy(io.MultiWriter(tmp, digest), in); err != nil {
|
||||
_ = tmp.Close()
|
||||
return "", fmt.Errorf("copy file: %w", err)
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
_ = tmp.Close()
|
||||
return "", fmt.Errorf("sync temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmpPath, perm); err != nil {
|
||||
return "", fmt.Errorf("chmod temp file: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, dst); err != nil {
|
||||
return "", fmt.Errorf("install temp file: %w", err)
|
||||
}
|
||||
removeTmp = false
|
||||
return hex.EncodeToString(digest.Sum(nil)), nil
|
||||
}
|
||||
145
internal/audio/s3_audio_test.go
Normal file
145
internal/audio/s3_audio_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
func TestMaterializeS3AudioCacheMissDownloadsAndPopulatesCache(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fake := &storage.FakeBackend{}
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
|
||||
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||
|
||||
result, err := MaterializeS3Audio(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||
}
|
||||
if !result.Downloaded || result.CacheHit {
|
||||
t.Fatalf("result = %#v, want downloaded miss", result)
|
||||
}
|
||||
if len(fake.Downloads) != 1 {
|
||||
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||
}
|
||||
assertFileEquals(t, req.DestPath, "audio")
|
||||
assertFileEquals(t, req.SpoolPath, "audio")
|
||||
assertFileEquals(t, result.CachePath, "audio")
|
||||
if result.Checksum == "" {
|
||||
t.Fatalf("checksum is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fake := &storage.FakeBackend{}
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||
if err != nil {
|
||||
t.Fatalf("cache path: %v", err)
|
||||
}
|
||||
writeAudioTestFile(t, cachePath, "audio")
|
||||
|
||||
result, err := MaterializeS3Audio(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||
}
|
||||
if !result.CacheHit || result.Downloaded {
|
||||
t.Fatalf("result = %#v, want cache hit", result)
|
||||
}
|
||||
if len(fake.Downloads) != 0 {
|
||||
t.Fatalf("downloads = %d, want 0", len(fake.Downloads))
|
||||
}
|
||||
assertFileEquals(t, req.DestPath, "audio")
|
||||
if result.SpoolPath != "" {
|
||||
t.Fatalf("spool path = %q, want empty on cache hit", result.SpoolPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeS3AudioInvalidCacheRefreshesFromS3(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fake := &storage.FakeBackend{}
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio")})
|
||||
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
|
||||
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
|
||||
if err != nil {
|
||||
t.Fatalf("cache path: %v", err)
|
||||
}
|
||||
writeAudioTestFile(t, cachePath, "stale")
|
||||
|
||||
result, err := MaterializeS3Audio(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeS3Audio() error = %v", err)
|
||||
}
|
||||
if !result.Downloaded || result.CacheHit {
|
||||
t.Fatalf("result = %#v, want refreshed miss", result)
|
||||
}
|
||||
if len(fake.Downloads) != 1 {
|
||||
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||
}
|
||||
assertFileEquals(t, req.DestPath, "fresh-audio")
|
||||
assertFileEquals(t, cachePath, "fresh-audio")
|
||||
}
|
||||
|
||||
func TestMaterializeS3AudioDownloadFailureLeavesDestinationMissing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fake := &storage.FakeBackend{DownloadErr: os.ErrPermission}
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
|
||||
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
|
||||
|
||||
_, err := MaterializeS3Audio(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "download s3 audio object") {
|
||||
t.Fatalf("error = %v, want download error", err)
|
||||
}
|
||||
assertMissing(t, req.DestPath)
|
||||
}
|
||||
|
||||
func testMaterializeRequest(t *testing.T, root string, fake *storage.FakeBackend, key string, size int64) S3MaterializeRequest {
|
||||
t.Helper()
|
||||
return S3MaterializeRequest{
|
||||
Store: fake,
|
||||
Object: storage.ObjectInfo{Key: key, Size: size, ETag: "etag"},
|
||||
Bucket: "my-dnd-archive",
|
||||
CacheRoot: filepath.Join(root, "cache"),
|
||||
CacheEnabled: true,
|
||||
SpoolPath: filepath.Join(root, "spool", "alice.flac"),
|
||||
DestPath: filepath.Join(root, "work", "audio", "alice.flac"),
|
||||
}
|
||||
}
|
||||
|
||||
func writeAudioTestFile(t *testing.T, path, contents string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %q: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatalf("write %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFileEquals(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("%q = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("stat %q = %v, want not exists", path, err)
|
||||
}
|
||||
}
|
||||
67
internal/config/cache_config_test.go
Normal file
67
internal/config/cache_config_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCacheDefaults(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||
root: /tmp/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
notification:
|
||||
timeout: 10s
|
||||
`, `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`)
|
||||
cfg, err := Load(pipelinePath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if cfg.Pipeline.Cache.Root != DefaultCacheRoot {
|
||||
t.Fatalf("cache.root = %q, want %q", cfg.Pipeline.Cache.Root, DefaultCacheRoot)
|
||||
}
|
||||
if cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio != DefaultCacheS3Audio {
|
||||
t.Fatalf("cache.s3_audio = %v, want %v", cfg.Pipeline.Cache.S3Audio, DefaultCacheS3Audio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||
root: /tmp/narratio
|
||||
cache:
|
||||
root: /var/cache/narratio
|
||||
unknown: true
|
||||
`, `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`)
|
||||
_, err := Load(pipelinePath, sessionPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("Load() error = %v, want strict decode failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheValidationRequiresRootWhenS3AudioEnabled(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
|
||||
root: /tmp/narratio
|
||||
cache:
|
||||
root: " "
|
||||
s3_audio: true
|
||||
`, `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`)
|
||||
cfg, err := Load(pipelinePath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
err = Validate(cfg)
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline.cache.root is required") {
|
||||
t.Fatalf("Validate() error = %v, want cache root error", err)
|
||||
}
|
||||
}
|
||||
137
internal/config/campaign_config_test.go
Normal file
137
internal/config/campaign_config_test.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCampaignConfigDefaultSearchOrder(t *testing.T) {
|
||||
want := []string{
|
||||
"/usr/local/etc/narratio/campaign.yml",
|
||||
"/etc/narratio/campaign.yml",
|
||||
}
|
||||
if len(DefaultCampaignConfigSearchPaths) != len(want) {
|
||||
t.Fatalf("DefaultCampaignConfigSearchPaths = %#v, want %#v", DefaultCampaignConfigSearchPaths, want)
|
||||
}
|
||||
for i := range want {
|
||||
if DefaultCampaignConfigSearchPaths[i] != want[i] {
|
||||
t.Fatalf("DefaultCampaignConfigSearchPaths[%d] = %q, want %q", i, DefaultCampaignConfigSearchPaths[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||
"campaign: sample-campaign\nunknown: true\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||
)
|
||||
|
||||
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("expected load error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCampaignSessionMergeFillsStableInputs(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||
)
|
||||
|
||||
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
if cfg.Session.Campaign != "sample-campaign" {
|
||||
t.Fatalf("session campaign = %q, want campaign config value", cfg.Session.Campaign)
|
||||
}
|
||||
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./campaign-speakers.yml", campaignPath, "campaign_config")
|
||||
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||
}
|
||||
|
||||
func TestCampaignSessionMergeSessionOverridesStableInputs(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||
"session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n speakers_file: ./session-speakers.yml\n",
|
||||
)
|
||||
|
||||
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./session-speakers.yml", sessionPath, "session_config")
|
||||
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||
}
|
||||
|
||||
func TestCampaignSessionMismatchFails(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||
"session_id: 2026-05-03\ncampaign: other-campaign\ninputs:\n audio_dir: ./audio\n",
|
||||
)
|
||||
|
||||
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("expected load error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not match campaign config") {
|
||||
t.Fatalf("error = %q, want campaign mismatch context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMissingCampaignFileFails(t *testing.T) {
|
||||
pipelinePath, _, sessionPath := writeCampaignConfigTestFiles(t,
|
||||
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||
)
|
||||
missingCampaignPath := filepath.Join(filepath.Dir(sessionPath), "missing-campaign.yml")
|
||||
|
||||
_, err := LoadWithSessionOptions(pipelinePath, missingCampaignPath, sessionPath, SessionLoadOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("expected load error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load campaign config") {
|
||||
t.Fatalf("error = %q, want campaign load context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func writeCampaignConfigTestFiles(t *testing.T, campaignYAML, sessionYAML string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
|
||||
pipelineYAML := "workspace:\n root: " + filepath.ToSlash(filepath.Join(dir, "work")) + "\nwhisperx:\n transcribe_url: https://example.com/transcribe\nanalyzer:\n timeout: 20m\nnotification:\n timeout: 10s\n"
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session.yml: %v", err)
|
||||
}
|
||||
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func assertResolvedStableInput(t *testing.T, got ResolvedInputFile, wantPath, wantConfigPath, wantSource string) {
|
||||
t.Helper()
|
||||
if got.Path != wantPath || got.ConfigPath != wantConfigPath || got.Source != wantSource {
|
||||
t.Fatalf("resolved input = %#v, want path=%q config_path=%q source=%q", got, wantPath, wantConfigPath, wantSource)
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,17 @@
|
||||
package config
|
||||
|
||||
// Config is the resolved combined configuration from pipeline.yml and session.yml.
|
||||
// Config is the resolved combined configuration from pipeline.yml,
|
||||
// campaign.yml, and session.yml.
|
||||
type Config struct {
|
||||
Pipeline *PipelineConfig
|
||||
Campaign *CampaignConfig
|
||||
Session *SessionConfig
|
||||
PipelinePath string
|
||||
CampaignPath string
|
||||
SessionPath string
|
||||
|
||||
StableInputs ResolvedStableInputs
|
||||
SessionSource SessionSource
|
||||
}
|
||||
|
||||
// PipelineConfig contains durable pipeline-level settings.
|
||||
@@ -13,6 +19,7 @@ type PipelineConfig struct {
|
||||
Workspace WorkspaceConfig `yaml:"workspace"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool"`
|
||||
Cache CacheConfig `yaml:"cache"`
|
||||
Archive *ArchiveConfig `yaml:"archive"`
|
||||
Secrets *SecretsConfig `yaml:"secrets"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||
@@ -25,6 +32,19 @@ type PipelineConfig struct {
|
||||
Notification NotificationConfig `yaml:"notification"`
|
||||
}
|
||||
|
||||
// CampaignConfig contains stable campaign-level identity and input defaults.
|
||||
type CampaignConfig struct {
|
||||
Campaign string `yaml:"campaign"`
|
||||
Inputs CampaignInputsConfig `yaml:"inputs"`
|
||||
}
|
||||
|
||||
// CampaignInputsConfig contains stable campaign-level input file references.
|
||||
type CampaignInputsConfig struct {
|
||||
SpeakersFile string `yaml:"speakers_file"`
|
||||
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||
GlossaryFile string `yaml:"glossary_file"`
|
||||
}
|
||||
|
||||
// SessionConfig contains per-session inputs and metadata.
|
||||
type SessionConfig struct {
|
||||
SessionID string `yaml:"session_id"`
|
||||
@@ -71,11 +91,18 @@ type SpoolConfig struct {
|
||||
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
||||
}
|
||||
|
||||
// CacheConfig configures durable local caches for reusable remote inputs.
|
||||
type CacheConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
S3Audio *bool `yaml:"s3_audio"`
|
||||
}
|
||||
|
||||
// ArchiveConfig configures archive behavior and artifact promotions.
|
||||
type ArchiveConfig struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
UploadRun *bool `yaml:"upload_run"`
|
||||
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
||||
Locks []ArchiveLockRule `yaml:"locks"`
|
||||
}
|
||||
|
||||
// ArchivePromotionRule configures one artifact promotion mapping.
|
||||
@@ -85,6 +112,18 @@ type ArchivePromotionRule struct {
|
||||
Required *bool `yaml:"required"`
|
||||
}
|
||||
|
||||
// ArchiveLockRule prevents one source-based promotion from overwriting its
|
||||
// top-level archive destination.
|
||||
type ArchiveLockRule struct {
|
||||
Source string `yaml:"source"`
|
||||
Reason string `yaml:"reason"`
|
||||
}
|
||||
|
||||
// ArchiveLockStore is the mutable per-session remote lock store.
|
||||
type ArchiveLockStore struct {
|
||||
Locks []ArchiveLockRule `yaml:"locks"`
|
||||
}
|
||||
|
||||
// WhisperXConfig configures WhisperX adapter settings.
|
||||
type WhisperXConfig struct {
|
||||
TranscribeURL string `yaml:"transcribe_url"`
|
||||
@@ -229,3 +268,29 @@ type SessionInputsConfig struct {
|
||||
type SessionAudioS3Input struct {
|
||||
Prefix string `yaml:"prefix"`
|
||||
}
|
||||
|
||||
// ResolvedStableInputs records where stable input file paths came from after
|
||||
// campaign/session merge.
|
||||
type ResolvedStableInputs struct {
|
||||
SpeakersFile ResolvedInputFile
|
||||
AutocorrectFile ResolvedInputFile
|
||||
GlossaryFile ResolvedInputFile
|
||||
}
|
||||
|
||||
// ResolvedInputFile records one merged config path and its source config file.
|
||||
type ResolvedInputFile struct {
|
||||
Path string
|
||||
ConfigPath string
|
||||
Source string
|
||||
}
|
||||
|
||||
// SessionSource records where session.yml came from before materialization.
|
||||
type SessionSource struct {
|
||||
Source string
|
||||
LocalPath string
|
||||
S3Bucket string
|
||||
S3Key string
|
||||
S3Size int64
|
||||
S3ETag string
|
||||
SpoolPath string
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package config
|
||||
|
||||
// Default filesystem locations for pipeline configuration lookup when --config
|
||||
// is omitted. Order is highest to lowest precedence.
|
||||
// Default filesystem locations for config lookup when config path flags are
|
||||
// omitted. Order is highest to lowest precedence.
|
||||
const (
|
||||
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
||||
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
||||
DefaultSessionConfigPathLocal = "./session.yml"
|
||||
DefaultCampaignConfigPathUsrLocal = "/usr/local/etc/narratio/campaign.yml"
|
||||
DefaultCampaignConfigPathEtc = "/etc/narratio/campaign.yml"
|
||||
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
||||
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
||||
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
|
||||
@@ -13,6 +14,8 @@ const (
|
||||
DefaultStorageS3RootPrefix = "dnd"
|
||||
DefaultWorkspaceRoot = "/var/lib/narratio"
|
||||
DefaultSpoolRoot = "/var/spool/narratio"
|
||||
DefaultCacheRoot = "/var/cache/narratio"
|
||||
DefaultCacheS3Audio = true
|
||||
|
||||
DefaultWhisperXLanguage = "en"
|
||||
DefaultWhisperXTimeout = "30m"
|
||||
@@ -88,13 +91,22 @@ var DefaultPipelineConfigSearchPaths = []string{
|
||||
DefaultPipelineConfigPathEtc,
|
||||
}
|
||||
|
||||
// DefaultCampaignConfigSearchPaths defines the default search order for
|
||||
// campaign.yml when callers do not provide an explicit path.
|
||||
//
|
||||
// Keep this in a variable so future defaults can be extended without changing
|
||||
// call sites.
|
||||
var DefaultCampaignConfigSearchPaths = []string{
|
||||
DefaultCampaignConfigPathUsrLocal,
|
||||
DefaultCampaignConfigPathEtc,
|
||||
}
|
||||
|
||||
// DefaultSessionConfigSearchPaths defines the default search order for
|
||||
// session.yml when callers do not provide an explicit path.
|
||||
//
|
||||
// Keep this in a variable so future defaults can be extended without changing
|
||||
// call sites.
|
||||
var DefaultSessionConfigSearchPaths = []string{
|
||||
DefaultSessionConfigPathLocal,
|
||||
DefaultSessionConfigPathUsrLocal,
|
||||
DefaultSessionConfigPathEtc,
|
||||
}
|
||||
|
||||
@@ -22,6 +22,15 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
||||
func LoadCampaign(path string) (*CampaignConfig, error) {
|
||||
var cfg CampaignConfig
|
||||
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("load campaign config: %w", err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// LoadSession loads session configuration from a YAML file with strict field checking.
|
||||
func LoadSession(path string) (*SessionConfig, error) {
|
||||
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
||||
@@ -40,20 +49,25 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
||||
}
|
||||
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
|
||||
}
|
||||
|
||||
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
|
||||
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
|
||||
// strict field checking after template rendering.
|
||||
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
|
||||
rendered, err := renderSessionTemplate(string(data), opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load session config: %w", err)
|
||||
}
|
||||
|
||||
var cfg SessionConfig
|
||||
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
|
||||
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
|
||||
return nil, fmt.Errorf("load session config: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
||||
return nil, fmt.Errorf(
|
||||
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
||||
path,
|
||||
label,
|
||||
strings.TrimSpace(opts.SessionID),
|
||||
strings.TrimSpace(cfg.SessionID),
|
||||
)
|
||||
@@ -63,7 +77,7 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
||||
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
||||
return nil, fmt.Errorf(
|
||||
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
||||
path,
|
||||
label,
|
||||
strings.TrimSpace(opts.PreviousSessionID),
|
||||
strings.TrimSpace(cfg.PreviousSessionID),
|
||||
)
|
||||
@@ -71,32 +85,171 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// Load loads and resolves combined pipeline and session configuration.
|
||||
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
||||
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
|
||||
// LoadArchiveLockStoreBytes loads a mutable session lock store with strict
|
||||
// field checking and source validation.
|
||||
func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
|
||||
var store ArchiveLockStore
|
||||
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
||||
return nil, fmt.Errorf("load archive lock store: %w", err)
|
||||
}
|
||||
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load archive lock store: %w", err)
|
||||
}
|
||||
store.Locks = locks
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// LoadWithSessionOptions loads and resolves combined pipeline and session
|
||||
// configuration with session template options.
|
||||
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||
// MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
|
||||
func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
|
||||
if store == nil {
|
||||
store = &ArchiveLockStore{}
|
||||
}
|
||||
data, err := yaml.Marshal(store)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal archive lock store: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
||||
// Passing only a session path is supported for package-internal compatibility;
|
||||
// in that form campaign.yml is expected next to the session file.
|
||||
func Load(pipelinePath string, paths ...string) (*Config, error) {
|
||||
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
}
|
||||
|
||||
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
|
||||
// session configuration with session template options.
|
||||
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||
pipelineCfg, err := LoadPipeline(pipelinePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
campaignCfg, err := LoadCampaign(campaignPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
||||
Source: "session_config",
|
||||
LocalPath: sessionPath,
|
||||
})
|
||||
}
|
||||
|
||||
// Resolve builds final stage-facing configuration from already loaded
|
||||
// pipeline, campaign, and session documents.
|
||||
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
||||
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(sessionSource.Source) == "" {
|
||||
sessionSource.Source = "session_config"
|
||||
}
|
||||
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
||||
sessionSource.LocalPath = sessionPath
|
||||
}
|
||||
|
||||
return &Config{
|
||||
Pipeline: pipelineCfg,
|
||||
Session: sessionCfg,
|
||||
PipelinePath: pipelinePath,
|
||||
SessionPath: sessionPath,
|
||||
Pipeline: pipelineCfg,
|
||||
Campaign: campaignCfg,
|
||||
Session: sessionCfg,
|
||||
PipelinePath: pipelinePath,
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
StableInputs: stableInputs,
|
||||
SessionSource: sessionSource,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
||||
switch len(paths) {
|
||||
case 1:
|
||||
sessionPath = paths[0]
|
||||
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
|
||||
case 2:
|
||||
campaignPath = paths[0]
|
||||
sessionPath = paths[1]
|
||||
default:
|
||||
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
|
||||
}
|
||||
return campaignPath, sessionPath, nil
|
||||
}
|
||||
|
||||
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
|
||||
if campaignCfg == nil {
|
||||
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
|
||||
}
|
||||
if sessionCfg == nil {
|
||||
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
|
||||
}
|
||||
|
||||
campaignName := strings.TrimSpace(campaignCfg.Campaign)
|
||||
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
|
||||
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
|
||||
return ResolvedStableInputs{}, fmt.Errorf(
|
||||
"campaign/session config invalid: session campaign %q does not match campaign config %q",
|
||||
sessionCampaign,
|
||||
campaignName,
|
||||
)
|
||||
}
|
||||
if sessionCampaign == "" {
|
||||
sessionCfg.Campaign = campaignName
|
||||
}
|
||||
|
||||
stable := ResolvedStableInputs{
|
||||
SpeakersFile: selectStableInput(
|
||||
campaignCfg.Inputs.SpeakersFile,
|
||||
sessionCfg.Inputs.SpeakersFile,
|
||||
campaignPath,
|
||||
sessionPath,
|
||||
),
|
||||
AutocorrectFile: selectStableInput(
|
||||
campaignCfg.Inputs.AutocorrectFile,
|
||||
sessionCfg.Inputs.AutocorrectFile,
|
||||
campaignPath,
|
||||
sessionPath,
|
||||
),
|
||||
GlossaryFile: selectStableInput(
|
||||
campaignCfg.Inputs.GlossaryFile,
|
||||
sessionCfg.Inputs.GlossaryFile,
|
||||
campaignPath,
|
||||
sessionPath,
|
||||
),
|
||||
}
|
||||
|
||||
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
|
||||
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
|
||||
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
|
||||
return stable, nil
|
||||
}
|
||||
|
||||
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
|
||||
if strings.TrimSpace(sessionValue) != "" {
|
||||
return ResolvedInputFile{
|
||||
Path: sessionValue,
|
||||
ConfigPath: sessionPath,
|
||||
Source: "session_config",
|
||||
}
|
||||
}
|
||||
return ResolvedInputFile{
|
||||
Path: campaignValue,
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
}
|
||||
}
|
||||
|
||||
func decodeStrictYAML(kind, path string, out any) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
@@ -208,6 +361,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
||||
applyWorkspaceDefaults(&cfg.Workspace)
|
||||
applyStorageDefaults(&cfg.Storage)
|
||||
applySpoolDefaults(&cfg.Spool)
|
||||
applyCacheDefaults(&cfg.Cache)
|
||||
applyArchiveDefaults(&cfg.Archive)
|
||||
applyWhisperXDefaults(&cfg.WhisperX)
|
||||
applySeriatimDefaults(&cfg.Seriatim)
|
||||
@@ -256,6 +410,18 @@ func applySpoolDefaults(cfg *SpoolConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyCacheDefaults(cfg *CacheConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultCacheRoot
|
||||
}
|
||||
if cfg.S3Audio == nil {
|
||||
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
|
||||
}
|
||||
}
|
||||
|
||||
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
|
||||
@@ -904,6 +904,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
||||
Report: boolPtr(true),
|
||||
},
|
||||
},
|
||||
Campaign: &CampaignConfig{Campaign: "sample-campaign"},
|
||||
Session: &SessionConfig{
|
||||
SessionID: "2026-05-03",
|
||||
Campaign: "sample-campaign",
|
||||
@@ -963,6 +964,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
|
||||
campaignPath := filepath.Join(examplesDir, "campaign.yml")
|
||||
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
|
||||
|
||||
var (
|
||||
@@ -972,7 +974,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
|
||||
cfg, err = Load(pipelinePath, sessionPath)
|
||||
} else {
|
||||
cfg, err = LoadWithSessionOptions(pipelinePath, sessionPath, tt.sessionOpts)
|
||||
cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, tt.sessionOpts)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("load example config error = %v", err)
|
||||
@@ -1001,14 +1003,34 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
}
|
||||
campaignYAML := `campaign: ` + campaignNameFromSessionYAML(sessionYAML) + `
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session.yml: %v", err)
|
||||
}
|
||||
|
||||
return pipelinePath, sessionPath
|
||||
}
|
||||
|
||||
func campaignNameFromSessionYAML(sessionYAML string) string {
|
||||
for _, line := range strings.Split(sessionYAML, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "campaign:") {
|
||||
return strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "campaign:")), `"'`)
|
||||
}
|
||||
}
|
||||
return "sample-campaign"
|
||||
}
|
||||
|
||||
@@ -241,3 +241,38 @@ inputs:
|
||||
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSessionBytesWithOptionsUsesSameTemplateAndStrictDecode(t *testing.T) {
|
||||
sessionYAML := []byte(`session_id: "{{ session_id }}"
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
|
||||
cfg, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", sessionYAML, SessionLoadOptions{SessionID: "2026-05-03"})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadSessionBytesWithOptions() error = %v", err)
|
||||
}
|
||||
if cfg.SessionID != "2026-05-03" {
|
||||
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||
}
|
||||
|
||||
_, err = LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\nunknown: true\n"), SessionLoadOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("expected strict decode error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("error = %q, want strict decode context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSessionBytesWithOptionsMismatchFails(t *testing.T) {
|
||||
_, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\n"), SessionLoadOptions{SessionID: "2026-04-04"})
|
||||
if err == nil {
|
||||
t.Fatal("expected mismatch error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "session_id mismatch") {
|
||||
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,6 +287,100 @@ archive:
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLockValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pipelineYML string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "valid built in source",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
reason: reviewed transcript
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "valid configured source",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "missing source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
locks:
|
||||
- reason: no source
|
||||
`,
|
||||
wantErr: "pipeline.archive.locks[0].source is required",
|
||||
},
|
||||
{
|
||||
name: "invalid source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.unknown
|
||||
`,
|
||||
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
|
||||
},
|
||||
{
|
||||
name: "duplicate source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
- source: " narratio.transcript.trimmed "
|
||||
`,
|
||||
wantErr: "duplicates another archive lock source",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
|
||||
cfg, err := Load(pipelinePath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
err = Validate(cfg)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
dest: transcripts/trimmed.json
|
||||
`
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||
_, err := Load(pipelinePath, sessionPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("Load() error = %v, want strict decode failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
@@ -301,6 +395,54 @@ archive:
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||
store, err := LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
reason: reviewed
|
||||
`), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
||||
}
|
||||
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.trimmed" || store.Locks[0].Reason != "reviewed" {
|
||||
t.Fatalf("locks = %#v", store.Locks)
|
||||
}
|
||||
|
||||
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
dest: transcripts/trimmed.json
|
||||
`), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
||||
}
|
||||
|
||||
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.trimmed
|
||||
- source: narratio.transcript.trimmed
|
||||
`), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicates another archive lock source") {
|
||||
t.Fatalf("duplicate error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
|
||||
merged := MergeArchiveLockRules(
|
||||
[]ArchiveLockRule{{Source: "narratio.transcript.trimmed", Reason: "static"}},
|
||||
[]ArchiveLockRule{
|
||||
{Source: "narratio.transcript.trimmed", Reason: "remote"},
|
||||
{Source: "narratio.transcript.full", Reason: "remote full"},
|
||||
},
|
||||
)
|
||||
if len(merged) != 2 {
|
||||
t.Fatalf("merged len = %d, want 2: %#v", len(merged), merged)
|
||||
}
|
||||
if merged[0].Source != "narratio.transcript.trimmed" || merged[0].Reason != "static" {
|
||||
t.Fatalf("merged[0] = %#v, want static lock", merged[0])
|
||||
}
|
||||
if merged[1].Source != "narratio.transcript.full" {
|
||||
t.Fatalf("merged[1] = %#v, want remote full lock", merged[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionAudioS3Validation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -17,6 +17,9 @@ func Validate(cfg *Config) error {
|
||||
if cfg.Pipeline == nil {
|
||||
return fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
if cfg.Campaign == nil {
|
||||
return fmt.Errorf("campaign config is required")
|
||||
}
|
||||
if cfg.Session == nil {
|
||||
return fmt.Errorf("session config is required")
|
||||
}
|
||||
@@ -24,6 +27,9 @@ func Validate(cfg *Config) error {
|
||||
if err := validatePipeline(cfg.Pipeline); err != nil {
|
||||
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
||||
}
|
||||
if err := validateCampaign(cfg.Campaign); err != nil {
|
||||
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
|
||||
}
|
||||
if err := validateSession(cfg.Session); err != nil {
|
||||
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
||||
}
|
||||
@@ -34,6 +40,16 @@ func Validate(cfg *Config) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCampaign(cfg *CampaignConfig) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("campaign config is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Campaign) == "" {
|
||||
return fmt.Errorf("campaign.campaign is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePipeline(cfg *PipelineConfig) error {
|
||||
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
||||
return fmt.Errorf("pipeline.workspace.root is required")
|
||||
@@ -47,6 +63,9 @@ func validatePipeline(cfg *PipelineConfig) error {
|
||||
if err := validateSpool(cfg.Spool); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateCache(cfg.Cache); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -104,6 +123,13 @@ func validateSpool(cfg SpoolConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCache(cfg CacheConfig) error {
|
||||
if cfg.S3Audio != nil && *cfg.S3Audio && strings.TrimSpace(cfg.Root) == "" {
|
||||
return fmt.Errorf("pipeline.cache.root is required when pipeline.cache.s3_audio is true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
@@ -136,9 +162,69 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
||||
}
|
||||
seenDest[normalizedDest] = struct{}{}
|
||||
}
|
||||
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Locks = locks
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateArchiveLockRules validates and normalizes source-based archive locks.
|
||||
func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumConfig, label string) ([]ArchiveLockRule, error) {
|
||||
seenLocks := map[string]struct{}{}
|
||||
out := make([]ArchiveLockRule, 0, len(locks))
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = "archive.locks"
|
||||
}
|
||||
for i, item := range locks {
|
||||
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
|
||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
if _, ok := seenLocks[source]; ok {
|
||||
return nil, fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
|
||||
}
|
||||
seenLocks[source] = struct{}{}
|
||||
out = append(out, ArchiveLockRule{
|
||||
Source: source,
|
||||
Reason: strings.TrimSpace(item.Reason),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MergeArchiveLockRules returns the union of static and remote locks. Static
|
||||
// locks win when both sources contain the same lock.
|
||||
func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []ArchiveLockRule {
|
||||
out := make([]ArchiveLockRule, 0, len(staticLocks)+len(remoteLocks))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range staticLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
for _, item := range remoteLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[source]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
||||
trimmed := strings.TrimSpace(source)
|
||||
switch trimmed {
|
||||
|
||||
@@ -23,6 +23,7 @@ type InputRecord struct {
|
||||
S3Size int64 `json:"s3_size,omitempty"`
|
||||
S3ETag string `json:"s3_etag,omitempty"`
|
||||
SpoolPath string `json:"spool_path,omitempty"`
|
||||
CachePath string `json:"cache_path,omitempty"`
|
||||
}
|
||||
|
||||
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
||||
|
||||
@@ -126,7 +126,14 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
||||
}
|
||||
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
|
||||
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
|
||||
sessionPaths,
|
||||
m,
|
||||
runtimeCatalog,
|
||||
env.Config.Pipeline.Archive.PromoteArtifacts,
|
||||
env.Config.Pipeline.Archive.Locks,
|
||||
sessionPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
||||
}
|
||||
@@ -166,6 +173,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
promotedUploaded,
|
||||
previousUploaded,
|
||||
skippedOptional,
|
||||
lockedPromotions,
|
||||
currentManifestKey,
|
||||
))
|
||||
if err != nil {
|
||||
@@ -204,6 +212,8 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
"previous_files_uploaded": len(previousUploaded),
|
||||
"previous_uploaded_paths": previousUploaded,
|
||||
"skipped_optional_promotions": skippedOptional,
|
||||
"locked_promotion_count": len(lockedPromotions),
|
||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||
"current_manifest_key": currentManifestKey,
|
||||
"current_run_id_key": currentRunPointerKey,
|
||||
"current_pointer_written": true,
|
||||
@@ -220,6 +230,16 @@ type archivePromotion struct {
|
||||
Provenance string
|
||||
}
|
||||
|
||||
type archiveLockedPromotion struct {
|
||||
Source string
|
||||
Dest string
|
||||
RemoteKey string
|
||||
Reason string
|
||||
Required bool
|
||||
LocalPath string
|
||||
Provenance string
|
||||
}
|
||||
|
||||
func archiveDisabled(env *Env) bool {
|
||||
cfg := env.Config.Pipeline.Archive
|
||||
if cfg == nil {
|
||||
@@ -319,26 +339,53 @@ func resolveArchivePromotions(
|
||||
m *manifest.Manifest,
|
||||
catalog *artifacts.ArtifactCatalog,
|
||||
rules []config.ArchivePromotionRule,
|
||||
) ([]archivePromotion, []string, error) {
|
||||
locks []config.ArchiveLockRule,
|
||||
sessionPrefix string,
|
||||
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
|
||||
out := make([]archivePromotion, 0, len(rules))
|
||||
skippedOptional := make([]string, 0)
|
||||
lockedPromotions := make([]archiveLockedPromotion, 0)
|
||||
lockSet := archiveLockSet(locks)
|
||||
for _, rule := range rules {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
required := rule.Required == nil || *rule.Required
|
||||
dest, err := resolveArchivePromotionDest(rule, catalog)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||
}
|
||||
lock, locked := lockSet[source]
|
||||
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
||||
if err != nil {
|
||||
if locked {
|
||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||
Source: source,
|
||||
Dest: dest,
|
||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||
Reason: strings.TrimSpace(lock.Reason),
|
||||
Required: required,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
||||
skippedOptional = append(skippedOptional, dest)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||
}
|
||||
if locked {
|
||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||
Source: source,
|
||||
Dest: dest,
|
||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||
Reason: strings.TrimSpace(lock.Reason),
|
||||
Required: required,
|
||||
LocalPath: resolved.Path,
|
||||
Provenance: resolved.Provenance,
|
||||
})
|
||||
continue
|
||||
}
|
||||
out = append(out, archivePromotion{
|
||||
Source: source,
|
||||
@@ -348,7 +395,21 @@ func resolveArchivePromotions(
|
||||
Provenance: resolved.Provenance,
|
||||
})
|
||||
}
|
||||
return out, skippedOptional, nil
|
||||
return out, skippedOptional, lockedPromotions, nil
|
||||
}
|
||||
|
||||
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||
for _, lock := range locks {
|
||||
source := strings.TrimSpace(lock.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
lock.Source = source
|
||||
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||
out[source] = lock
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
||||
@@ -663,6 +724,7 @@ func archiveMetadataPreview(
|
||||
promotedUploaded []string,
|
||||
previousUploaded []string,
|
||||
skippedOptional []string,
|
||||
lockedPromotions []archiveLockedPromotion,
|
||||
currentManifestKey string,
|
||||
) map[string]any {
|
||||
return map[string]any{
|
||||
@@ -677,9 +739,27 @@ func archiveMetadataPreview(
|
||||
"previous_files_uploaded": len(previousUploaded),
|
||||
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
||||
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
||||
"locked_promotion_count": len(lockedPromotions),
|
||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||
"current_manifest_key": currentManifestKey,
|
||||
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||
"current_pointer_written": false,
|
||||
"audio_upload_skipped": true,
|
||||
}
|
||||
}
|
||||
|
||||
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(locked))
|
||||
for _, item := range locked {
|
||||
out = append(out, map[string]any{
|
||||
"source": item.Source,
|
||||
"dest": item.Dest,
|
||||
"remote_key": item.RemoteKey,
|
||||
"reason": item.Reason,
|
||||
"required": item.Required,
|
||||
"local_path": item.LocalPath,
|
||||
"provenance": item.Provenance,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -215,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
||||
env, m, _ := archiveFixture(t)
|
||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
|
||||
}
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
|
||||
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||
if _, ok := fake.Objects[trimmedKey]; ok {
|
||||
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
|
||||
}
|
||||
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
|
||||
if _, ok := fake.Objects[recapKey]; !ok {
|
||||
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
|
||||
}
|
||||
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
|
||||
if _, ok := fake.Objects[runTrimmedKey]; !ok {
|
||||
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
|
||||
}
|
||||
|
||||
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
|
||||
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
|
||||
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
|
||||
}
|
||||
|
||||
if result.Metadata["promoted_files_uploaded"] != 1 {
|
||||
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
|
||||
}
|
||||
if result.Metadata["locked_promotion_count"] != 1 {
|
||||
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
|
||||
}
|
||||
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||
if len(locked) != 1 {
|
||||
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||
}
|
||||
if locked[0]["source"] != "narratio.transcript.trimmed" ||
|
||||
locked[0]["dest"] != "transcripts/trimmed.json" ||
|
||||
locked[0]["remote_key"] != trimmedKey ||
|
||||
locked[0]["reason"] != "human reviewed" ||
|
||||
locked[0]["required"] != true ||
|
||||
locked[0]["local_path"] == "" ||
|
||||
locked[0]["provenance"] == "" {
|
||||
t.Fatalf("locked promotion metadata = %#v", locked[0])
|
||||
}
|
||||
|
||||
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
|
||||
var current map[string]any
|
||||
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, ¤t); err != nil {
|
||||
t.Fatalf("unmarshal current manifest: %v", err)
|
||||
}
|
||||
stages := current["stages"].(map[string]any)
|
||||
archive := stages["archive"].(map[string]any)
|
||||
meta := archive["metadata"].(map[string]any)
|
||||
if meta["locked_promotion_count"] != float64(1) {
|
||||
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
|
||||
}
|
||||
items := meta["locked_promotions"].([]any)
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
||||
env, m, _ := archiveFixture(t)
|
||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
|
||||
}
|
||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
|
||||
}
|
||||
|
||||
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
|
||||
if _, ok := fake.Objects[mergedKey]; ok {
|
||||
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
|
||||
}
|
||||
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
||||
t.Fatalf("current run pointer should be written for locked missing promotion")
|
||||
}
|
||||
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||
if len(locked) != 1 {
|
||||
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||
}
|
||||
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
|
||||
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
||||
env, m, _ := archiveFixture(t)
|
||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||
{Source: "narratio.transcript.trimmed", Reason: "already published"},
|
||||
}
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
|
||||
|
||||
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
got := string(fake.Objects[trimmedKey].Data)
|
||||
if got != "previously published\n" {
|
||||
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
|
||||
}
|
||||
for _, upload := range fake.Uploads {
|
||||
if upload.Key == trimmedKey {
|
||||
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
||||
env, m, _ := archiveFixture(t)
|
||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||
|
||||
@@ -30,8 +30,10 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
||||
store := artifacts.NewLocalStore(root)
|
||||
cfgDir := t.TempDir()
|
||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||
writeStageTestFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
|
||||
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||
@@ -48,7 +50,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
||||
env := &Env{
|
||||
Config: &config.Config{
|
||||
SessionPath: sessionPath,
|
||||
CampaignPath: campaignPath,
|
||||
PipelinePath: pipelinePath,
|
||||
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Workspace: config.WorkspaceConfig{Root: root},
|
||||
Storage: config.StorageConfig{
|
||||
@@ -62,14 +66,28 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
||||
UploadRun: boolPtr(true),
|
||||
},
|
||||
},
|
||||
StableInputs: config.ResolvedStableInputs{
|
||||
SpeakersFile: config.ResolvedInputFile{
|
||||
Path: "./speakers.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
AutocorrectFile: config.ResolvedInputFile{
|
||||
Path: "./autocorrect.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
GlossaryFile: config.ResolvedInputFile{
|
||||
Path: "./glossary.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
},
|
||||
Session: &config.SessionConfig{
|
||||
SessionID: "2026-05-03",
|
||||
Campaign: "sample-campaign",
|
||||
Inputs: config.SessionInputsConfig{
|
||||
AudioDir: "./audio",
|
||||
SpeakersFile: "./speakers.yml",
|
||||
AutocorrectFile: "./autocorrect.yml",
|
||||
GlossaryFile: "./glossary.yml",
|
||||
AudioDir: "./audio",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/audio"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -25,6 +26,7 @@ func (prepareStage) Name() string { return "prepare" }
|
||||
func (prepareStage) Declares() IODecl {
|
||||
return IODecl{
|
||||
Inputs: []artifacts.Ref{
|
||||
{Kind: "config", Category: "inputs", RelativePath: "campaign.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
||||
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
||||
@@ -59,21 +61,30 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
||||
}
|
||||
|
||||
campaignSrc := env.Config.CampaignPath
|
||||
if err := requireFile(campaignSrc, "campaign.yml"); err != nil {
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
|
||||
sessionSrc := env.Config.SessionPath
|
||||
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
sessionDir := filepath.Dir(sessionSrc)
|
||||
|
||||
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile)
|
||||
speakersInput := stableInputSource(env.Config.StableInputs.SpeakersFile, env.Config.Session.Inputs.SpeakersFile, sessionSrc)
|
||||
autocorrectInput := stableInputSource(env.Config.StableInputs.AutocorrectFile, env.Config.Session.Inputs.AutocorrectFile, sessionSrc)
|
||||
glossaryInput := stableInputSource(env.Config.StableInputs.GlossaryFile, env.Config.Session.Inputs.GlossaryFile, sessionSrc)
|
||||
|
||||
speakersSrc, err := resolveConfigRelativePath(speakersInput)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
||||
}
|
||||
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile)
|
||||
autocorrectSrc, err := resolveConfigRelativePath(autocorrectInput)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
||||
}
|
||||
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile)
|
||||
glossarySrc, err := resolveConfigRelativePath(glossaryInput)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
||||
}
|
||||
@@ -96,17 +107,44 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
||||
}
|
||||
|
||||
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
|
||||
inputs := make([]manifest.InputRecord, 0, 6+len(resolvedLocalAudio))
|
||||
registerInput := func(kind, path, checksum string) {
|
||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
||||
}
|
||||
registerConfigInput := func(kind, path, checksum, source string) {
|
||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum, Source: source})
|
||||
}
|
||||
registerSessionConfigInput := func(path, checksum string) {
|
||||
source := env.Config.SessionSource
|
||||
if strings.TrimSpace(source.Source) == "" {
|
||||
source.Source = "session_config"
|
||||
}
|
||||
inputs = append(inputs, manifest.InputRecord{
|
||||
Kind: "session_config",
|
||||
Path: path,
|
||||
Checksum: checksum,
|
||||
Source: source.Source,
|
||||
S3Bucket: source.S3Bucket,
|
||||
S3Key: source.S3Key,
|
||||
S3Size: source.S3Size,
|
||||
S3ETag: source.S3ETag,
|
||||
SpoolPath: source.SpoolPath,
|
||||
})
|
||||
}
|
||||
|
||||
campaignDst := filepath.Join(paths.InputsDir, "campaign.yml")
|
||||
campaignChecksum, err := copyFileIfChanged(env.ArtifactStore, campaignSrc, campaignDst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize campaign.yml: %w", err)
|
||||
}
|
||||
registerConfigInput("campaign_config", campaignDst, campaignChecksum, "campaign_config")
|
||||
|
||||
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
||||
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
||||
}
|
||||
registerInput("session_config", sessionDst, sessionChecksum)
|
||||
registerSessionConfigInput(sessionDst, sessionChecksum)
|
||||
|
||||
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
||||
if err != nil {
|
||||
@@ -120,25 +158,29 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
|
||||
|
||||
for _, cfgFile := range []struct {
|
||||
kind string
|
||||
src string
|
||||
dst string
|
||||
kind string
|
||||
src string
|
||||
dst string
|
||||
source string
|
||||
}{
|
||||
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")},
|
||||
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")},
|
||||
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")},
|
||||
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml"), source: speakersInput.Source},
|
||||
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml"), source: autocorrectInput.Source},
|
||||
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml"), source: glossaryInput.Source},
|
||||
} {
|
||||
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
||||
}
|
||||
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
||||
registerConfigInput(cfgFile.kind, cfgFile.dst, checksum, cfgFile.source)
|
||||
}
|
||||
|
||||
var audioCacheStats s3AudioMaterializationStats
|
||||
if useS3Audio {
|
||||
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
|
||||
stats, err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
|
||||
}
|
||||
audioCacheStats = stats
|
||||
} else {
|
||||
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
@@ -173,6 +215,11 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
"inputs_count": len(inputs),
|
||||
"audio_files_resolved": countAudioInputs(inputs),
|
||||
}
|
||||
if useS3Audio {
|
||||
metadata["audio_cache_hits"] = audioCacheStats.CacheHits
|
||||
metadata["audio_cache_misses"] = audioCacheStats.CacheMisses
|
||||
metadata["audio_s3_downloads"] = audioCacheStats.Downloads
|
||||
}
|
||||
if len(previousRequirements) > 0 {
|
||||
metadata["previous_requirements_count"] = len(previousRequirements)
|
||||
if previousHydration != nil {
|
||||
@@ -195,6 +242,31 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
|
||||
return yaml.Marshal(cfg)
|
||||
}
|
||||
|
||||
func stableInputSource(resolved config.ResolvedInputFile, fallbackPath, fallbackConfigPath string) config.ResolvedInputFile {
|
||||
if strings.TrimSpace(resolved.Path) != "" || strings.TrimSpace(resolved.ConfigPath) != "" || strings.TrimSpace(resolved.Source) != "" {
|
||||
if strings.TrimSpace(resolved.ConfigPath) == "" {
|
||||
resolved.ConfigPath = fallbackConfigPath
|
||||
}
|
||||
if strings.TrimSpace(resolved.Source) == "" {
|
||||
resolved.Source = "session_config"
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
return config.ResolvedInputFile{
|
||||
Path: fallbackPath,
|
||||
ConfigPath: fallbackConfigPath,
|
||||
Source: "session_config",
|
||||
}
|
||||
}
|
||||
|
||||
func resolveConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||
basePath := strings.TrimSpace(input.ConfigPath)
|
||||
if basePath == "" {
|
||||
return "", fmt.Errorf("source config path is required")
|
||||
}
|
||||
return resolvePath(filepath.Dir(basePath), input.Path)
|
||||
}
|
||||
|
||||
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
||||
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
||||
if inputs.AudioS3 != nil {
|
||||
@@ -282,28 +354,34 @@ func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolve
|
||||
return nil
|
||||
}
|
||||
|
||||
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
|
||||
type s3AudioMaterializationStats struct {
|
||||
CacheHits int
|
||||
CacheMisses int
|
||||
Downloads int
|
||||
}
|
||||
|
||||
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) (s3AudioMaterializationStats, error) {
|
||||
if env.ObjectStore == nil {
|
||||
return fmt.Errorf("s3 audio input requires object store backend")
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires object store backend")
|
||||
}
|
||||
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
||||
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
|
||||
}
|
||||
|
||||
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||
if campaign == "" {
|
||||
return fmt.Errorf("session campaign is required for s3 audio input")
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("session campaign is required for s3 audio input")
|
||||
}
|
||||
runID := strings.TrimSpace(m.RunID)
|
||||
if runID == "" {
|
||||
return fmt.Errorf("run id is required for s3 audio input")
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
|
||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||
@@ -321,7 +399,7 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return audioObjects[i].Key < audioObjects[j].Key
|
||||
})
|
||||
if len(audioObjects) == 0 {
|
||||
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
}
|
||||
|
||||
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||
@@ -331,45 +409,60 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
|
||||
|
||||
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
|
||||
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
|
||||
}
|
||||
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
|
||||
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||
}
|
||||
|
||||
seenBase := map[string]string{}
|
||||
stats := s3AudioMaterializationStats{}
|
||||
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
|
||||
for _, obj := range audioObjects {
|
||||
base := path.Base(obj.Key)
|
||||
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
||||
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||
}
|
||||
seenBase[base] = obj.Key
|
||||
|
||||
spoolPath := filepath.Join(spoolAudioDir, base)
|
||||
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
|
||||
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
|
||||
}
|
||||
|
||||
workPath := filepath.Join(workAudioDir, base)
|
||||
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
|
||||
result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||
Store: env.ObjectStore,
|
||||
Object: obj,
|
||||
Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
||||
CacheRoot: strings.TrimSpace(env.Config.Pipeline.Cache.Root),
|
||||
CacheEnabled: cacheEnabled,
|
||||
SpoolPath: spoolPath,
|
||||
DestPath: workPath,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
|
||||
return s3AudioMaterializationStats{}, err
|
||||
}
|
||||
if result.CacheHit {
|
||||
stats.CacheHits++
|
||||
} else {
|
||||
stats.CacheMisses++
|
||||
}
|
||||
if result.Downloaded {
|
||||
stats.Downloads++
|
||||
}
|
||||
|
||||
*inputs = append(*inputs, manifest.InputRecord{
|
||||
Kind: "audio",
|
||||
Path: workPath,
|
||||
Checksum: checksum,
|
||||
Checksum: result.Checksum,
|
||||
Source: "s3",
|
||||
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
||||
S3Key: obj.Key,
|
||||
S3Size: obj.Size,
|
||||
S3ETag: obj.ETag,
|
||||
SpoolPath: spoolPath,
|
||||
SpoolPath: result.SpoolPath,
|
||||
CachePath: result.CachePath,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func countAudioInputs(inputs []manifest.InputRecord) int {
|
||||
|
||||
@@ -50,6 +50,10 @@ func TestHydratePreviousSessionArtifactsDownloadsManifestAndRequiredArtifact(t *
|
||||
if _, err := os.Stat(recapPath); err != nil {
|
||||
t.Fatalf("previous artifact missing: %v", err)
|
||||
}
|
||||
nestedRecapPath := filepath.Join(sessionPaths.PreviousArtifactsDir, "artifacts", "session_recap.md")
|
||||
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
|
||||
}
|
||||
|
||||
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
|
||||
if manifestInput == nil {
|
||||
|
||||
@@ -35,6 +35,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
||||
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
for _, p := range []string{
|
||||
filepath.Join(paths.InputsDir, "campaign.yml"),
|
||||
filepath.Join(paths.InputsDir, "session.yml"),
|
||||
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
||||
filepath.Join(paths.InputsDir, "speakers.yml"),
|
||||
@@ -48,8 +49,8 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(m.Inputs) != 7 {
|
||||
t.Fatalf("manifest inputs len = %d, want 7", len(m.Inputs))
|
||||
if len(m.Inputs) != 8 {
|
||||
t.Fatalf("manifest inputs len = %d, want 8", len(m.Inputs))
|
||||
}
|
||||
for _, in := range m.Inputs {
|
||||
if in.Checksum == "" {
|
||||
@@ -151,6 +152,64 @@ func TestPrepareStageIdempotent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareStageRecordsLocalSessionProvenance(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
root := filepath.Dir(env.Config.SessionPath)
|
||||
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||
if sessionInput.Source != "session_config" {
|
||||
t.Fatalf("session source = %q, want session_config", sessionInput.Source)
|
||||
}
|
||||
if sessionInput.S3Bucket != "" || sessionInput.S3Key != "" || sessionInput.SpoolPath != "" {
|
||||
t.Fatalf("local session input has unexpected remote provenance: %#v", sessionInput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareStageRecordsRemoteSessionProvenance(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
remoteSessionPath := filepath.Join(t.TempDir(), "downloaded-session.yml")
|
||||
writeFile(t, remoteSessionPath, "session_id: 2026-05-03\ninputs:\n audio_s3:\n prefix: audio/\n")
|
||||
env.Config.SessionPath = remoteSessionPath
|
||||
env.Config.Session.Inputs.AudioDir = ""
|
||||
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||
env.Config.SessionSource = config.SessionSource{
|
||||
Source: "session_config.s3",
|
||||
LocalPath: remoteSessionPath,
|
||||
S3Bucket: "my-dnd-archive",
|
||||
S3Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml",
|
||||
S3Size: 58,
|
||||
S3ETag: "session-etag",
|
||||
SpoolPath: remoteSessionPath,
|
||||
}
|
||||
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||
env.ObjectStore = fake
|
||||
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||
if sessionInput.Source != "session_config.s3" {
|
||||
t.Fatalf("session source = %q, want session_config.s3", sessionInput.Source)
|
||||
}
|
||||
if sessionInput.S3Bucket != "my-dnd-archive" || sessionInput.S3Key != "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml" {
|
||||
t.Fatalf("remote session input missing bucket/key: %#v", sessionInput)
|
||||
}
|
||||
if sessionInput.S3Size != 58 || sessionInput.S3ETag != "session-etag" || sessionInput.SpoolPath != remoteSessionPath {
|
||||
t.Fatalf("remote session input missing metadata: %#v", sessionInput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
env.Config.Session.Campaign = "forsaken"
|
||||
@@ -186,6 +245,15 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||
t.Fatalf("expected file %q: %v", p, err)
|
||||
}
|
||||
}
|
||||
if got := result.Metadata["audio_cache_hits"]; got != 0 {
|
||||
t.Fatalf("audio_cache_hits = %#v, want 0", got)
|
||||
}
|
||||
if got := result.Metadata["audio_cache_misses"]; got != 2 {
|
||||
t.Fatalf("audio_cache_misses = %#v, want 2", got)
|
||||
}
|
||||
if got := result.Metadata["audio_s3_downloads"]; got != 2 {
|
||||
t.Fatalf("audio_s3_downloads = %#v, want 2", got)
|
||||
}
|
||||
|
||||
audioInputs := 0
|
||||
for _, in := range m.Inputs {
|
||||
@@ -199,15 +267,75 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||
if in.S3Bucket != "my-dnd-archive" {
|
||||
t.Fatalf("audio input bucket = %q", in.S3Bucket)
|
||||
}
|
||||
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
|
||||
if in.S3Key == "" || in.SpoolPath == "" || in.CachePath == "" || in.Checksum == "" {
|
||||
t.Fatalf("audio input missing provenance: %#v", in)
|
||||
}
|
||||
if _, err := os.Stat(in.CachePath); err != nil {
|
||||
t.Fatalf("expected cache path %q: %v", in.CachePath, err)
|
||||
}
|
||||
}
|
||||
if audioInputs != 2 {
|
||||
t.Fatalf("audio input count = %d, want 2", audioInputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareStageS3AudioUsesCacheOnRerun(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
env.Config.Session.Campaign = "forsaken"
|
||||
env.Config.Session.Inputs.AudioDir = ""
|
||||
env.Config.Session.Inputs.AudioFiles = nil
|
||||
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||
env.ObjectStore = fake
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("first prepare.Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Downloads) != 1 {
|
||||
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(filepath.Join(m.LocalWorkDir, "audio")); err != nil {
|
||||
t.Fatalf("remove work audio: %v", err)
|
||||
}
|
||||
if err := os.RemoveAll(m.LocalSpoolDir); err != nil {
|
||||
t.Fatalf("remove spool audio: %v", err)
|
||||
}
|
||||
fake.DownloadErr = os.ErrPermission
|
||||
|
||||
m2 := manifest.New("2026-05-03", time.Now().UTC())
|
||||
m2.RunID = "20260515T041522Z-a1b2c3d4"
|
||||
m2.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m2.SessionID, m2.RunID)
|
||||
m2.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m2.SessionID, m2.RunID)
|
||||
result, err := (prepareStage{}).Run(context.Background(), env, m2)
|
||||
if err != nil {
|
||||
t.Fatalf("cached prepare.Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Downloads) != 1 {
|
||||
t.Fatalf("downloads = %d, want cached rerun to avoid new download", len(fake.Downloads))
|
||||
}
|
||||
if got := result.Metadata["audio_cache_hits"]; got != 1 {
|
||||
t.Fatalf("audio_cache_hits = %#v, want 1", got)
|
||||
}
|
||||
if got := result.Metadata["audio_s3_downloads"]; got != 0 {
|
||||
t.Fatalf("audio_s3_downloads = %#v, want 0", got)
|
||||
}
|
||||
audioInput := findManifestInput(t, m2.Inputs, "audio")
|
||||
if audioInput.CachePath == "" {
|
||||
t.Fatalf("audio input missing cache path: %#v", audioInput)
|
||||
}
|
||||
if audioInput.SpoolPath != "" {
|
||||
t.Fatalf("audio input spool path = %q, want empty on cache hit", audioInput.SpoolPath)
|
||||
}
|
||||
mustReadFileEquals(t, filepath.Join(m2.LocalWorkDir, "audio", "alice.flac"), "alice")
|
||||
}
|
||||
|
||||
func TestPrepareStageS3AudioFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -382,6 +510,10 @@ func TestPrepareStageHydratesRequiredPreviousArtifactAndRecordsInputs(t *testing
|
||||
if _, err := os.Stat(recapPath); err != nil {
|
||||
t.Fatalf("expected previous artifact: %v", err)
|
||||
}
|
||||
nestedRecapPath := filepath.Join(paths.PreviousArtifactsDir, "artifacts", "session_recap.md")
|
||||
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
|
||||
}
|
||||
|
||||
var hasPreviousManifest, hasPreviousArtifact bool
|
||||
for _, in := range m.Inputs {
|
||||
@@ -473,25 +605,51 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
||||
|
||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||
|
||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||
writeFile(t, campaignPath, `campaign: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`)
|
||||
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Workspace: config.WorkspaceConfig{Root: workspace},
|
||||
Cache: config.CacheConfig{Root: filepath.Join(t.TempDir(), "cache"), S3Audio: boolPtr(true)},
|
||||
},
|
||||
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||
SessionPath: sessionPath,
|
||||
CampaignPath: campaignPath,
|
||||
PipelinePath: pipelinePath,
|
||||
Session: &config.SessionConfig{
|
||||
SessionID: "2026-05-03",
|
||||
Campaign: "sample-campaign",
|
||||
Inputs: config.SessionInputsConfig{
|
||||
AudioDir: "./audio",
|
||||
SpeakersFile: "./speakers.yml",
|
||||
AutocorrectFile: "./autocorrect.yml",
|
||||
GlossaryFile: "./glossary.yml",
|
||||
AudioDir: "./audio",
|
||||
},
|
||||
},
|
||||
StableInputs: config.ResolvedStableInputs{
|
||||
SpeakersFile: config.ResolvedInputFile{
|
||||
Path: "./speakers.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
AutocorrectFile: config.ResolvedInputFile{
|
||||
Path: "./autocorrect.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
GlossaryFile: config.ResolvedInputFile{
|
||||
Path: "./glossary.yml",
|
||||
ConfigPath: campaignPath,
|
||||
Source: "campaign_config",
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -519,3 +677,25 @@ func snapshotInputs(inputs []manifest.InputRecord) map[string]string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func findManifestInput(t *testing.T, inputs []manifest.InputRecord, kind string) manifest.InputRecord {
|
||||
t.Helper()
|
||||
for _, input := range inputs {
|
||||
if input.Kind == kind {
|
||||
return input
|
||||
}
|
||||
}
|
||||
t.Fatalf("manifest input kind %q not found in %#v", kind, inputs)
|
||||
return manifest.InputRecord{}
|
||||
}
|
||||
|
||||
func mustReadFileEquals(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("%q = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,8 +233,10 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
||||
|
||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||
writeFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||
@@ -243,7 +245,9 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
||||
concurrency := 2
|
||||
cfg := &config.Config{
|
||||
PipelinePath: pipelinePath,
|
||||
CampaignPath: campaignPath,
|
||||
SessionPath: sessionPath,
|
||||
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Workspace: config.WorkspaceConfig{Root: workspace},
|
||||
WhisperX: config.WhisperXConfig{
|
||||
@@ -265,6 +269,11 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
||||
GlossaryFile: "./glossary.yml",
|
||||
},
|
||||
},
|
||||
StableInputs: config.ResolvedStableInputs{
|
||||
SpeakersFile: config.ResolvedInputFile{Path: "./speakers.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||
AutocorrectFile: config.ResolvedInputFile{Path: "./autocorrect.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||
GlossaryFile: config.ResolvedInputFile{Path: "./glossary.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||
},
|
||||
}
|
||||
|
||||
env := &Env{
|
||||
|
||||
Reference in New Issue
Block a user