29 Commits

Author SHA1 Message Date
083c01cfa0 Implemented narratio analyze as a shortcut to run the analyze stage only
Some checks failed
ci/woodpecker/tag/release Pipeline failed
2026-05-21 23:02:19 -05:00
2937696024 Add clean command 2026-05-21 22:48:18 -05:00
b817a5b772 Implemented shared S3 audio caching for prepare and restore --include-audio 2026-05-21 22:22:08 -05:00
3022f20beb Simplified the output of narratio artifacts list --remote and narratio status --session-id 2026-05-21 21:20:41 -05:00
ca1ded1821 Bugfix for commands that list artifacts in the S3 backend 2026-05-21 21:00:51 -05:00
3752f3ed28 Added remote artifact listing to narratio status 2026-05-21 20:49:28 -05:00
870c2d69d5 Consolidated addition, removal, and listing of locks under a single narratio locks command 2026-05-21 19:36:14 -05:00
135407ba7c Implemented a centralized secret-backed object-store helper 2026-05-21 19:13:10 -05:00
228c348e42 Implemented operations helper commands for validation, locking, and status 2026-05-21 11:50:20 -05:00
a813bd5a50 Fixed a redundant path bug for previous session artifacts 2026-05-21 10:58:49 -05:00
d8f58dce31 Normalize the default configuration discovery paths for all three config files, and update documentation and tests accordingly 2026-05-21 09:55:56 -05:00
7111edeca4 Add archive promotion locks 2026-05-20 21:40:09 -05:00
3aae4bbb12 Add remote session loading 2026-05-20 20:55:13 -05:00
b29d8eeb50 Add campaign configuration support 2026-05-20 20:41:28 -05:00
dffb432537 Removed completed roadmap for previous_session artifacts 2026-05-20 20:15:47 -05:00
2dd38c7913 Refine campaign and remote session roadmap 2026-05-20 20:15:05 -05:00
bc2ade38d9 Finalize previous-session artifact documentation and restore-analyze continuity coverage 2026-05-20 15:17:04 +00:00
5be831eb13 Restore archived previous-session cache files with session state 2026-05-20 15:05:48 +00:00
cae4d99a89 Archive durable previous-session cache files with session state 2026-05-20 15:03:30 +00:00
e09dc0512d Add analyze integration coverage for previous-session inputs 2026-05-20 15:01:27 +00:00
ae82bc1ce0 Resolve canonical previous-session artifact sources from prepared previous cache 2026-05-20 14:59:28 +00:00
01eb7aa1aa Add prepare rerun guidance for unresolved previous-session analyze inputs 2026-05-20 14:55:44 +00:00
2ca700195c Integrate previous-session artifact hydration into prepare stage 2026-05-20 14:53:22 +00:00
2b08c34539 Add prepare helper to hydrate previous-session artifacts from archive 2026-05-20 14:49:22 +00:00
79f1fc1e09 Add helper to collect previous-session artifact input requirements 2026-05-20 14:37:02 +00:00
9c753270bd Add canonical previous-session artifact source parsing and validation 2026-05-20 14:34:23 +00:00
b907cb01aa Add previous-session workspace path helpers and layout support 2026-05-20 14:31:03 +00:00
7824afd4a5 Add previous session ID templating and CLI support 2026-05-20 14:26:32 +00:00
2a4e1e912c Update documentation to include a roadmap for previous session artifact support 2026-05-20 09:12:17 -05:00
92 changed files with 9238 additions and 1341 deletions

View File

@@ -6,7 +6,9 @@
narratio run --session-id 2026-04-04
```
This command uses default discovery for `pipeline.yml` and `session.yml`; both files must be discoverable unless you pass explicit `--config` and `--session` paths.
This command uses default system discovery for `pipeline.yml`, `campaign.yml`, and local `session.yml`. If local session discovery misses and S3 storage is configured, `--session-id` can load remote `session.yml` from the canonical session prefix.
Default discovery checks system config locations only. Pass `--config`, `--campaign`, and `--session` to use files from the current working directory.
## Command Overview
@@ -15,9 +17,15 @@ Implemented commands:
- `run`: execute pipeline stages and persist manifest state.
- `plan`: validate config, prepare workspace layout, and print stage run/skip decisions.
- `resume`: continue from first non-succeeded stage unless forced.
- `status`: read and print stage statuses from an existing manifest.
- `status`: read an existing manifest or inspect local/remote state for a session.
- `run-stage`: execute exactly one stage.
- `analyze`: force-rerun the analyze stage.
- `restore`: restore durable local session state from the committed remote archive state.
- `session validate`: run read-only preflight checks for a session.
- `session init`: create local or remote `session.yml`.
- `artifacts list`: list effective artifact source IDs.
- `locks`: list, add, and remove archive promotion locks.
- `clean`: remove local workspace/spool state for one session or all local sessions.
Unknown commands print usage and exit non-zero.
@@ -28,35 +36,54 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
### `run`
- `--config <path>`: optional explicit `pipeline.yml` path.
- `--campaign <path>`: optional explicit `campaign.yml` path.
- `--session <path>`: optional explicit `session.yml` path.
- `--session-id <value>`: session template variable value.
- `--previous-session-id <value>`: previous-session template variable value.
- `--force`: force stage execution.
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
### `plan`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--force`
### `resume`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--force`
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
### `run-stage`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--force`
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
- positional `<stage>`: required stage name.
### `analyze`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
`analyze` is force-by-design and does not accept `--force`.
Valid stage names:
- `prepare`
@@ -72,15 +99,76 @@ Valid stage names:
### `restore`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--dry-run`: plan restore actions without writing local files.
- `--force`: overwrite local conflicting files with remote archive files.
- `--include-audio`: include durable archived `audio/**` files in restore scope.
### `clean`
- `--session-id <value>`: required for session cleanup unless `--all` is set.
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--previous-session-id <value>`
- `--all`: clean all local session work/spool state using pipeline config only.
- `--dry-run`: print cleanup targets without deleting.
- `--clear-cache`: also remove matching S3 audio cache entries.
### `status`
- `--manifest <path>`: required manifest path.
- `--manifest <path>`: inspect one manifest file.
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
### `session validate`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
### `session init`
- `--config <path>`: required.
- `--campaign <path>`: required.
- `--session-id <value>`: required.
- `--output <path>`: local `session.yml` target; mutually exclusive with `--remote`.
- `--remote`: write remote `session.yml` to the canonical session prefix; mutually exclusive with `--output`.
- `--previous-session-id <value>`
- `--date <value>`
- `--title <value>`
- `--audio-s3-prefix <prefix>`: defaults to `audio/` when neither audio flag is provided.
- `--audio-dir <path>`: local audio directory; mutually exclusive with `--audio-s3-prefix`.
- `--force`: overwrite existing local or remote target.
### `artifacts list`
- `--config <path>`
- `--campaign <path>`
- `--session <path>`
- `--session-id <value>`
- `--previous-session-id <value>`
- `--remote`: check remote availability for configured archive promotion destinations.
### `locks`
- `--session-id <value>`: required for list, add, and remove.
- `--config <path>`: optional explicit `pipeline.yml` path.
- `--campaign <path>`: optional explicit `campaign.yml` path.
- `--session <path>`: optional explicit `session.yml` path.
- `--previous-session-id <value>`: optional session template value.
- `add <source>`: add a remote lock for one artifact or transcript source.
- `add --reason <text>`: record an optional remote lock reason.
- `add --force`: update the reason for an existing remote lock.
- `remove <source>`: remove one remote lock.
## Command Reference
@@ -92,14 +180,15 @@ Purpose:
Syntax:
```bash
narratio run [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>]
narratio run [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
```
Success output:
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
Common failure cases:
- missing default config/session paths when flags omitted.
- missing system default config/campaign/session paths when flags omitted.
- missing local session plus missing/unavailable remote `session.yml`.
- invalid template/rendered session mismatch.
- unknown/invalid `--artifacts` value.
- `--artifacts` with unknown configured artifact key.
@@ -112,7 +201,7 @@ Purpose:
Syntax:
```bash
narratio plan [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force]
narratio plan [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
```
Success output includes:
@@ -121,7 +210,8 @@ Success output includes:
- `totals: run=<n> skip=<n>`
Common failure cases:
- same config/session discovery and validation failures as `run`.
- same config/campaign/session discovery and validation failures as `run`.
- remote session fallback failures when local session discovery misses.
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
### `resume`
@@ -132,7 +222,7 @@ Purpose:
Syntax:
```bash
narratio resume [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>]
narratio resume [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
```
Success output:
@@ -147,22 +237,111 @@ Common failure cases:
### `status`
Purpose:
- Inspect one manifest file without executing stages.
- Inspect one manifest file, or inspect configured local/remote state for a session.
Syntax:
```bash
narratio status --manifest <manifest.json>
narratio status [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
```
Success output includes:
Manifest output includes:
- `session_id: <id>`
- `updated_at: <timestamp>`
- `stages:` entries (`- <stage>: <status>`)
Session output includes:
- session ID, campaign, workspace, session config source.
- local manifest state when present.
- remote current archive state when storage is configured.
- catalog-based remote output availability for expected transcript and artifact sources.
- effective archive locks and conservative next actions.
Common failure cases:
- missing `--manifest`.
- missing `--manifest` when no config/session flags are provided.
- unreadable or invalid manifest path.
- invalid config or remote session fallback failure in session mode.
### `session validate`
Purpose:
- Run read-only preflight checks for a session.
Syntax:
```bash
narratio session validate [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>]
```
Checks include:
- effective config and session source.
- stable input files.
- local or remote audio availability.
- previous-session requirements.
- archive promotions and effective locks.
Warnings do not fail the command. Any `ERROR` finding exits non-zero.
### `session init`
Purpose:
- Create a strict-decoded session skeleton locally or in object storage.
Syntax:
```bash
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --output ./session.yml
narratio session init --config <pipeline.yml> --campaign <campaign.yml> --session-id <id> --remote
```
Behavior:
- exactly one of `--output` or `--remote` is required.
- remote writes target `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
- existing local or remote targets fail unless `--force` is passed.
- remote writes use existence checks, not compare-and-swap.
### `artifacts list`
Purpose:
- List built-in, configured, previous-session, promoted, and locked artifact sources.
Syntax:
```bash
narratio artifacts list [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--remote]
```
`--remote` checks promoted top-level object availability through the storage adapter. Remote markers appear only in the `Promoted` section, which reports each configured archive promotion destination and includes `dest=<path>` when that destination differs from the source's canonical path.
### `locks`
Purpose:
- Inspect and mutate source-based archive promotion locks for one session.
Syntax:
```bash
narratio locks --session-id <id>
narratio locks add --session-id <id> [--reason <text>] [--force] <source>
narratio locks remove --session-id <id> <source>
```
Behavior:
- `--session-id` is required for list, add, and remove.
- optional `--config`, `--campaign`, and `--session` override default config discovery.
- list mode prints effective locks from static `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`.
- `locks add` writes only the remote lock store and fails if the source is already locked by pipeline config.
- `locks remove` removes only remote locks and cannot remove static pipeline locks.
- `locks add --force` is required to update an existing remote lock reason.
Examples:
```bash
narratio locks --session-id 2026-04-04
narratio locks add --session-id 2026-04-04 --reason "manual transcript review" narratio.transcript.trimmed
narratio locks remove --session-id 2026-04-04 narratio.transcript.trimmed
```
### `run-stage`
@@ -172,7 +351,7 @@ Purpose:
Syntax:
```bash
narratio run-stage [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
narratio run-stage [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
```
Success output:
@@ -188,15 +367,35 @@ Common failure cases:
- unknown stage name.
- using `--artifacts` with any non-`analyze` stage.
### `restore`
### `analyze`
Purpose:
- Restore durable session state (`manifest.json`, `transcripts/**`, `artifacts/**`, and optional `audio/**`) from the committed remote archive current state.
- Force-rerun the analyze stage.
- Provide a shorter equivalent for `narratio run-stage --force analyze`.
Syntax:
```bash
narratio restore [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--dry-run] [--force] [--include-audio]
narratio analyze [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
```
Success output:
- `narratio analyze: executed=<n> skipped=<n> force=true; manifest=<path>`
Common failure cases:
- positional arguments.
- `--force`, because force is implicit.
- unknown configured artifact keys.
### `restore`
Purpose:
- Restore durable session state (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, and optional `audio/**`) from the committed remote archive current state.
Syntax:
```bash
narratio restore [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
```
Success output (dry-run):
@@ -221,6 +420,40 @@ Common failure cases:
- local conflicts without `--force`.
- session lock conflict.
When `--include-audio` is set, S3 audio files are restored through the shared audio cache. Cache hits avoid re-downloading large audio objects.
### `clean`
Purpose:
- Remove local Narratio work/spool state for testing, reruns, or recovery from corrupted local files.
- Preserve durable S3 audio cache state unless `--clear-cache` is passed.
Syntax:
```bash
narratio clean --session-id <id> [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--clear-cache]
narratio clean --all [--config <pipeline.yml>] [--dry-run] [--clear-cache]
```
Session cleanup deletes:
- `{workspace.root}/work/{campaign}/{session_id}`
- `{spool.root}/{campaign}/{session_id}`
All-session cleanup deletes:
- `{workspace.root}/work`
- the contents of `{spool.root}`, while preserving the spool root directory itself.
Cache behavior:
- cache is preserved by default.
- `--clear-cache` in session mode removes cached S3 audio files for the resolved session.
- `--all --clear-cache` removes the configured Narratio S3 audio cache namespace for the configured bucket/root prefix.
- `--clear-cache` does not delete arbitrary files under `pipeline.cache.root`.
Common failure cases:
- missing `--session-id` when `--all` is not set.
- combining `--all` with `--campaign`, `--session`, `--session-id`, or `--previous-session-id`.
- unsafe cleanup target, such as a symlink, a non-directory session target, a configured root directory, or a path outside the configured root.
## Common Workflows
Default-discovery run:
@@ -241,10 +474,10 @@ Resume with selected analyze artifacts:
narratio resume --session-id 2026-04-04 --artifacts player_handout
```
Run only analyze stage with selected artifacts:
Force-rerun analyze with selected artifacts:
```bash
narratio run-stage --session-id 2026-04-04 --artifacts player_handout analyze
narratio analyze --session-id 2026-04-04 --artifacts player_handout
```
Preview restore actions without writes:
@@ -257,7 +490,27 @@ Restore and then force analyze:
```bash
narratio restore --session-id 2026-04-04
narratio run-stage --session-id 2026-04-04 --force analyze
narratio analyze --session-id 2026-04-04
```
Rehydrate canonical previous-session inputs after artifact-input changes:
```bash
narratio run-stage --session-id 2026-04-04 --force prepare
```
Reset local state before testing restore:
```bash
narratio clean --session-id 2026-04-04 --dry-run
narratio clean --session-id 2026-04-04
narratio restore --session-id 2026-04-04 --include-audio
```
Clean all local sessions while keeping cached S3 audio:
```bash
narratio clean --all
```
## Diagnostic / Recovery Commands
@@ -269,7 +522,7 @@ narratio status --manifest <manifest.json>
```
Get manifest path from previous output:
- `run`, `resume`, and `run-stage` print `manifest=<path>` on success.
- `run`, `resume`, `run-stage`, and `analyze` print `manifest=<path>` on success.
## `--artifacts` and `--force`

View File

@@ -2,12 +2,13 @@
## 1. Overview
Narratio loads two YAML files:
Narratio loads three YAML files:
- `pipeline.yml`: pipeline-level runtime configuration.
- `session.yml`: per-session metadata and input selection.
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
These commands load and validate both files before running:
These commands load and validate all three files before running:
- `narratio run`
- `narratio plan`
@@ -19,12 +20,23 @@ Behavior:
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
- session templates render before session YAML decode.
- remote `session.yml` uses the same strict decode and template behavior as local `session.yml`.
- defaults are applied for optional pipeline fields.
- campaign-level stable input paths fill missing session input paths.
- session-level stable input paths override campaign-level input paths.
- validation enforces required fields, value formats, and cross-field constraints.
## 2. Config file discovery
Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
These commands use the same config discovery behavior:
- `narratio run`
- `narratio plan`
- `narratio resume`
- `narratio run-stage`
- `narratio restore`
Pipeline config lookup:
- if `--config <path>` is provided, that path is used.
- if omitted, Narratio searches in order:
@@ -32,27 +44,44 @@ Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
2. `/etc/narratio/pipeline.yml`
- first existing file wins.
## 3. Session file discovery and templating
Campaign config lookup:
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
- if `--session <path>` is provided, that path is used.
- if `--campaign <path>` is provided, that path is used.
- if omitted, Narratio searches in order:
1. `./session.yml`
2. `/usr/local/etc/narratio/session.yml`
3. `/etc/narratio/session.yml`
1. `/usr/local/etc/narratio/campaign.yml`
2. `/etc/narratio/campaign.yml`
- first existing file wins.
Template behavior:
Session config lookup:
- if `--session <path>` is provided, that path is used.
- if `--session` is omitted, Narratio searches locally in order:
1. `/usr/local/etc/narratio/session.yml`
2. `/etc/narratio/session.yml`
- first existing local file wins.
- if no local session file is found, `--session-id <value>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
- local discovery always runs before remote fallback.
- local files in the current working directory are used only when passed explicitly, for example `--config ./pipeline.yml --campaign ./campaign.yml --session ./session.yml`.
## 3. Session templating
Template behavior for local and remote `session.yml`:
- supported placeholders:
- `{{session_id}}`
- `{{ session_id }}`
- `{{previous_session_id}}`
- `{{ previous_session_id }}`
- `--session-id <value>` supplies the placeholder value.
- `--previous-session-id <value>` supplies the previous-session placeholder value.
- unresolved placeholders fail load.
- if rendered `session_id` mismatches `--session-id`, load fails.
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
## 4. Minimal pipeline config
## 4. Minimal config set
### `pipeline.yml`
```yaml
whisperx:
@@ -65,25 +94,58 @@ Why this is sufficient:
- `workspace.root` defaults to `/var/lib/narratio`.
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
## 5. Minimal session template
### `campaign.yml`
```yaml
campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
```
Why this is sufficient:
- `campaign` supplies the stable campaign identity.
- stable input files are required and resolve relative to `campaign.yml` when copied during `prepare`.
### `session.yml`
```yaml
session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml
```
Usage:
Why this is sufficient:
- `session_id` is required and can be rendered from `--session-id`.
- `campaign` can be omitted because it is supplied by `campaign.yml`.
- stable input paths can be omitted because `campaign.yml` supplies defaults.
- local `audio_dir` resolves relative to `session.yml`.
Minimal local-file usage:
```bash
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03
```
## 6. Production-oriented config
Previous-session-enabled variant:
```yaml
session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
inputs:
audio_dir: ./audio
```
```bash
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
```
## 5. Production-oriented config set
### `pipeline.yml`
```yaml
workspace:
@@ -103,6 +165,10 @@ spool:
root: /var/spool/narratio
delete_audio_after_archive: true
cache:
root: /var/cache/narratio
s3_audio: true
archive:
enabled: true
upload_run: true
@@ -113,6 +179,9 @@ archive:
- source: narratio.artifact.session_recap
dest: artifacts/session_recap.md
required: true
locks:
- source: narratio.artifact.session_recap
reason: Final recap was manually edited.
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
@@ -127,16 +196,58 @@ scriptorium:
transcript:
source: narratio.transcript.trimmed
required: true
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
```
### `campaign.yml`
```yaml
campaign: forsaken
inputs:
speakers_file: /srv/narratio/campaigns/forsaken/speakers.yml
autocorrect_file: /srv/narratio/campaigns/forsaken/autocorrect.yml
glossary_file: /srv/narratio/campaigns/forsaken/glossary.yml
```
### Local `session.yml`
```yaml
session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
date: 2026-05-03
title: The Black Cabin
inputs:
audio_s3:
prefix: audio/
```
### S3-first session config
For S3-first operation, upload the same `session.yml` content to:
```text
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml
```
Then run with explicit or discovered pipeline/campaign config and no `--session`:
```bash
narratio run --config /usr/local/etc/narratio/pipeline.yml --campaign /usr/local/etc/narratio/campaign.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
```
Operational notes:
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
- `source` is required; `dest` is optional and derived when omitted.
- `archive.locks` skips top-level promotion overwrites for static locked sources while preserving run-local uploads.
- operator-created mutable locks are stored at `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml` and are merged with static locks.
- Narratio does not auto-promote all generated analyze artifacts.
- `restore` reads the same config/session inputs and restore scope is bounded by committed archive current state.
- `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
- `clean` removes workspace/spool state by default and preserves `pipeline.cache.root` unless `--clear-cache` is passed.
## 7. Full pipeline reference
## 6. Full pipeline reference
| Path | Type | Required | Default |
| --- | --- | --- | --- |
@@ -155,12 +266,17 @@ Operational notes:
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
| `pipeline.cache.s3_audio` | bool | No | `true` |
| `pipeline.archive.enabled` | bool | No | `true` |
| `pipeline.archive.upload_run` | bool | No | `true` |
| `pipeline.archive.promote_artifacts[]` | list | No | trimmed transcript rule |
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
| `pipeline.archive.locks[]` | list | No | empty |
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
| `pipeline.archive.locks[].reason` | string | No | empty |
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
| `pipeline.whisperx.language` | string | No | `en` |
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
@@ -243,13 +359,14 @@ Scriptorium artifact-key and dependency rules:
Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
- `previous_session_artifact`
- `narratio.previous_session.artifact.<configured_artifact_key>`
- `narratio.transcript.merged`
- `narratio.transcript.polished`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
- `narratio.bounds.session`
- `narratio.artifact.<configured_artifact_key>`
- `previous_session_artifact` (legacy path-based source; uses `inputs.<key>.path`)
`pipeline.archive.promote_artifacts[].source` values:
@@ -260,6 +377,8 @@ Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
- `narratio.bounds.session`
- `narratio.artifact.<configured_artifact_key>`
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
Archive promotion destination rules:
- `dest` must be a clean relative path (not absolute, no traversal).
@@ -269,25 +388,57 @@ Archive promotion destination rules:
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
- derivation failure is a config validation error.
Archive lock rules:
- locks are source-based and do not accept `dest`.
- duplicate lock sources are rejected.
- static `pipeline.archive.locks` win over remote mutable locks for the same source.
- locked promotions are recorded as intentional skips in archive metadata.
- locked required promotions do not fail archive by default.
- ordinary `--force` reruns do not override locks.
Remote mutable lock store:
- path: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml`.
- strict YAML shape: top-level `locks`, each with `source` and optional `reason`.
- `narratio locks add` and `narratio locks remove` mutate only the remote lock store.
- writes use existence checks plus `--force` for updates; they are not compare-and-swap atomic.
Restore-related implications:
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
- restore scope considers only committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, optional `audio/**`).
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
- S3 audio downloads use `pipeline.spool.root` for active downloads and `pipeline.cache.root` for reusable cached audio when `pipeline.cache.s3_audio` is true.
- `pipeline.cache.root` is durable local cache state. It is not workspace state and is preserved by default by `narratio clean`.
## 7. Full campaign reference
| Path | Type | Required | Default |
| --- | --- | --- | --- |
| `campaign.campaign` | string | Yes | none |
| `campaign.inputs.speakers_file` | string | Yes | none |
| `campaign.inputs.autocorrect_file` | string | Yes | none |
| `campaign.inputs.glossary_file` | string | Yes | none |
Campaign input paths may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
## 8. Full session reference
| Path | Type | Required | Default |
| --- | --- | --- | --- |
| `session.session_id` | string | Yes | none |
| `session.campaign` | string | Yes | none |
| `session.previous_session_id` | string | No | empty |
| `session.campaign` | string | No | `campaign.campaign` |
| `session.date` | string | No | empty |
| `session.title` | string | No | empty |
| `session.inputs.audio_dir` | string | Conditional | empty |
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
| `session.inputs.speakers_file` | string | Yes | none |
| `session.inputs.autocorrect_file` | string | Yes | none |
| `session.inputs.glossary_file` | string | Yes | none |
| `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
| `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
| `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
Audio-source rule:
@@ -297,6 +448,11 @@ Audio-source rule:
- `audio_s3.prefix`
- `audio_s3` cannot be combined with local audio fields.
Previous-session rule:
- if `session.previous_session_id` is set, it must not equal `session.session_id`.
- canonical previous-session sources (`narratio.previous_session.artifact.<name>`) are hydrated during `prepare` from archive current state when required by enabled configured artifacts.
## 9. Secrets
Narratio supports filesystem-based secret injection via `pipeline.secrets.env_dir`.
@@ -323,6 +479,7 @@ Maintained examples:
- `examples/pipeline.minimal.yml`
- `examples/pipeline.production.yml`
- `examples/pipeline.full.annotated.yml`
- `examples/campaign.yml`
- `examples/session.template.yml`
- `examples/session.local-audio.yml`
- `examples/session.s3-audio.yml`

View File

@@ -69,6 +69,8 @@ For design principles and invariants, see [docs/architecture.md](./architecture.
2. Add or update command tests (`TestExecute` and command-specific tests).
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
### Add or modify stages/adapters
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.

View File

@@ -47,6 +47,7 @@ Current execution usage:
Default construction in app runner:
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
- Object-store construction goes through app command orchestration so configured filesystem secrets are loaded before the storage adapter is initialized.
- Callers can inject test/fake implementations through `app.RunOptions.Env`.
## State and manifest behavior

View File

@@ -1,41 +1,36 @@
# Internal: Artifacts
## Purpose
Define Narratio's artifact identity and resolution model for built-in transcript/bounds artifacts and runtime-configured analyze artifacts.
Define Narratio artifact identity, catalog, and source-resolution behavior for:
- built-in session artifacts;
- configured analyze artifacts;
- canonical previous-session artifact sources.
## Inputs and outputs
Inputs:
- artifact sources from config/runtime (`pipeline.scriptorium.artifacts.*.inputs.*.source`)
- session paths and optional session manifest stage outputs
- runtime artifact catalog state for configured artifact sources
- configured input sources (`pipeline.scriptorium.artifacts.*.inputs.*.source`);
- session paths and manifest inputs/outputs;
- runtime catalog state.
Outputs:
- resolved local artifact path and provenance (`ResolvedSessionArtifact`)
- runtime catalog entries for planned/executable/available artifacts
- validation errors for unsupported, missing, or invalid artifact sources
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
- runtime catalog entries for built-ins and configured artifacts;
- requirement sets for canonical previous-session inputs.
- canonical S3 session, run, current, session config, session locks, audio, and promoted artifact keys.
## Boundaries
Owns:
- built-in artifact registry and content validation rules
- runtime artifact catalog for configured artifact source IDs
- source resolution behavior for built-in and configured artifact sources
- built-in source registry and validation;
- configured artifact catalog identity (`narratio.artifact.<name>`);
- canonical previous-session source parsing and resolution;
- previous-session requirement collection (`CollectPreviousArtifactRequirements`).
Does not own:
- artifact generation (stages produce files)
- manifest transition policy
- archive promotion behavior
## Config fields used
- `pipeline.scriptorium.artifacts.<name>.enabled`
- `pipeline.scriptorium.artifacts.<name>.output_path`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
## External adapters used
- none
## State and manifest behavior
Built-in registry entries:
- prepare-stage remote hydration;
- stage success/skip transitions;
- archive upload orchestration.
## Built-in IDs
| Artifact ID | Canonical file | Producer stage | Output kind |
| --- | --- | --- | --- |
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` |
@@ -44,44 +39,68 @@ Built-in registry entries:
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` |
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` |
Runtime catalog entries include built-ins and configured `narratio.artifact.<name>` sources.
## Source families
- built-in: `narratio.transcript.*`, `narratio.bounds.session`
- configured artifact: `narratio.artifact.<artifact_key>`
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
Catalog states:
- `planned`: source is registered and known for this run
- `executable`: configured artifact is selected for analyze execution
- `available`: artifact has a usable file path (generated this run or reused from disk)
## S3 key helpers
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
- session config: `{session_prefix}/session.yml`
- session lock store: `{session_prefix}/locks.yml`
- run prefix: `{session_prefix}/runs/{run_id}/`
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
- current manifest: `{session_prefix}/current/manifest.json`
- current run pointer: `{session_prefix}/current/run_id.txt`
Resolution behavior:
- built-in sources resolve via manifest producer outputs first, then canonical fallback path
- configured `narratio.artifact.<name>` sources resolve through runtime catalog availability
- configured source lookup requires catalog context
## Runtime catalog model
Catalog entries track:
- `planned`: source is registered for this run;
- `executable`: configured artifact is selected for analyze execution;
- `available`: usable local file exists (generated this run or reused from disk).
Configured artifact provenance values:
Configured artifact provenance values include:
- `generated.current_analyze_run`
- `filesystem.disabled_artifact_output`
Content validation:
- transcript built-ins: JSON with top-level `segments` array
- bounds built-in: valid JSON
- configured artifacts: non-empty text file
Previous-session canonical provenance values include:
- `manifest.inputs.previous_cache`
- `current_session.previous_cache`
## Skip and resume behavior
- resolver and catalog have no direct skip/resume decisions
- stage/runner skip-resume behavior consumes catalog/resolver results
## Resolution behavior
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
- Archive-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
## Previous-session requirement scanning
`CollectPreviousArtifactRequirements`:
- scans enabled configured artifacts only;
- includes canonical previous-session sources only;
- deduplicates by artifact key;
- merges required/optional references (`required` wins);
- records deterministic sorted source locations for diagnostics.
## Validation behavior
- transcript built-ins: JSON with top-level `segments` array;
- bounds built-in: valid JSON;
- configured and previous-session artifact files: non-empty text content.
## Failure behavior
- unsupported source -> source validation error
- known source unavailable -> `ErrSessionArtifactNotFound`
- configured source without catalog -> resolution error
- resolved file with invalid content -> validation error
- unsupported source or malformed canonical previous source: validation/resolution error;
- known source unavailable: `ErrSessionArtifactNotFound`;
- configured/previous canonical source without catalog: error;
- resolved invalid file content: validation error.
## Tests to inspect before changing
- `internal/artifacts/artifact_resolver_test.go`
- `internal/artifacts/catalog_test.go`
- `internal/artifacts/previous_requirements_test.go`
- `internal/stage/prepare_previous_test.go`
- `internal/stage/analyze_test.go`
- `internal/config/scriptorium_test.go`
## Architectural invariants
- built-in IDs are static and registry-backed
- configured artifact IDs are runtime-derived (`narratio.artifact.<name>`) and catalog-backed
- built-in/source resolution remains deterministic and validation-gated
- Built-in source IDs are static.
- Configured and previous-session source IDs are artifact-key based and validation-gated.
- Resolution behavior remains deterministic and manifest-aware.

View File

@@ -1,11 +1,11 @@
# Internal: Command Restore
## Purpose
Define the implemented `narratio restore` command contract: committed remote-state discovery, deterministic planning, safe file installation, conflict policy, and restore reporting.
Define the implemented `narratio restore` contract: committed remote-state discovery, deterministic plan classification, safe file install semantics, and restore reporting.
## Inputs and outputs
Inputs:
- CLI flags: `--config`, `--session`, `--session-id`, `--dry-run`, `--force`, `--include-audio`.
- CLI flags: `--config`, `--session`, `--session-id`, `--previous-session-id`, `--dry-run`, `--force`, `--include-audio`.
- Resolved/validated `pipeline.yml` and `session.yml`.
- Configured remote object store.
- Remote committed current-state markers (`current/run_id.txt`, `current/manifest.json`).
@@ -34,6 +34,8 @@ Does not own:
- `pipeline.workspace.root` (local restore target root).
- `pipeline.storage.*` (remote backend + archive identity derivation).
- `pipeline.storage.s3.*` identity components used by archive prefix helpers.
- `pipeline.spool.root` for active audio downloads.
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
- `session.session_id`
- `session.campaign`
@@ -48,15 +50,32 @@ Does not own:
- `current/run_id.txt` must exist and be non-empty.
- `current/manifest.json` must decode and match requested session/campaign.
- Non-dry-run writes restore files to canonical session paths.
- With `--include-audio`, restore uses the shared S3 audio cache for `audio/**` objects. Cache hits avoid object downloads; cache misses download through spool, install the work file, and populate cache.
- Manifest install behavior:
- validated before replacement.
- installed last among download actions.
- existing local manifest is preserved if restored manifest validation/install fails.
- Non-dry-run report persists summary/action status metadata in `reports/restore-latest.json`.
Restore path scope:
- includes:
- `manifest.json`
- `transcripts/**`
- `artifacts/**`
- `previous/**`
- `audio/**` only when `--include-audio` is set
- excludes:
- `runs/**`
- `logs/**`
- `reports/**`
- `config/**`
- `inputs/**`
- remote `current/**` pointer files as local restore targets
## Skip and resume behavior
- Restore does not participate in stage skip/resume decisions.
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
- Audio cache is outside the workspace and is reused across restore and prepare invocations.
- Dry-run is read-only and returns plan output only.
## Failure behavior
@@ -80,7 +99,7 @@ Does not own:
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
- Local path mapping is traversal-safe and constrained to session root.
- Restore scope is deterministic and path-classified:
- include `manifest.json`, `transcripts/**`, `artifacts/**`
- include `manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`
- include `audio/**` only with `--include-audio`
- exclude `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`
- Command remains standalone; no implicit `run --restore` behavior.

View File

@@ -3,31 +3,36 @@
## Purpose
Execute selected configured Scriptorium artifacts in deterministic dependency order and promote successful outputs to canonical session artifact paths.
## Inputs and Outputs
## Inputs and outputs
Inputs:
- configured artifact definitions from `pipeline.scriptorium.artifacts`
- selected artifact filter from runtime (`--artifacts`) when provided
- resolved artifact input sources declared per artifact (`inputs.*.source`)
- optional previous-session file inputs (`previous_session_artifact`)
- configured artifact definitions from `pipeline.scriptorium.artifacts`;
- selected artifact filter (`--artifacts`) when provided;
- resolved artifact sources from resolver/catalog.
Source types used by analyze:
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`;
- configured artifacts: `narratio.artifact.<artifact_key>`;
- canonical previous-session artifacts: `narratio.previous_session.artifact.<artifact_key>`;
- legacy path-based previous-session source: `previous_session_artifact` (uses `inputs.*.path`).
Outputs:
- one promoted output file per executed configured artifact at that artifact's configured `output_path`
- stage metadata containing generated artifact entries and reused disabled-artifact entries
- promoted configured artifact files at each configured `output_path`;
- stage metadata (`generated_artifacts`, `reused_artifacts`, selected/order info).
## Boundaries
Owns:
- runtime artifact catalog construction for analyze execution
- selected-artifact planning and dependency ordering
- per-artifact input resolution, var resolution, timeout/render-debug resolution
- Scriptorium run/render invocation for each selected artifact
- run-local output generation and canonical promotion
- runtime artifact catalog construction;
- selected-artifact planning and dependency ordering;
- per-input resolution and required/optional handling;
- Scriptorium render/run invocation;
- run-local output generation and canonical promotion.
Does not own:
- transcript generation/processing stages
- archive promotion policy
- per-artifact resume semantics
- prepare-time previous-session hydration;
- object-store access for previous-session sources;
- archive promotion policy.
## Config Fields Used
## Config fields used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
@@ -36,49 +41,41 @@ Does not own:
- `pipeline.scriptorium.timeout`
- `pipeline.scriptorium.render_debug`
- `pipeline.scriptorium.artifacts.<name>.*`
- `enabled`
- `depends_on`
- `prompt_id`
- `profile_id`
- `timeout`
- `output_path`
- `render_debug`
- `inputs`
- `vars`
## External Adapters Used
## External adapters used
- Scriptorium adapter:
- optional `RenderArtifact` (render debug)
- `RunArtifact` (artifact generation)
- optional `RenderArtifact` when render-debug is enabled;
- `RunArtifact` for artifact generation.
## State and Manifest Behavior
- If `pipeline.scriptorium` is absent, stage returns success metadata with `skipped=true`.
- If no artifacts are configured, stage returns success metadata with `skipped=true`.
- If zero artifacts are executable after `enabled` + `--artifacts` filtering, stage returns success metadata with `skipped=true`.
- Builds runtime catalog with built-ins and configured artifacts.
- Non-executable configured artifacts are marked available only when their configured output file exists and is valid on disk.
- Executes selected configured artifacts in topological order with deterministic tie-breaking.
- For each generated artifact, records metadata fields including `name`, `source_id`, `output_kind`, `path`, `prompt_id`, `profile_id`, and `provenance`.
- Reused disabled artifacts are recorded separately in `reused_artifacts` with provenance `filesystem.disabled_artifact_output`.
## State and manifest behavior
- If Scriptorium config is absent, or no artifacts are executable after filtering, analyze returns success metadata with `skipped=true`.
- Builds runtime catalog with built-ins and configured `narratio.artifact.<name>` entries.
- Non-executable configured artifacts may still be marked available from existing canonical output files.
- Resolves canonical previous-session sources from local prepared `previous/` cache:
- prefers manifest-backed previous input paths when present;
- may fall back to current-session `previous/` filesystem paths.
- Analyze does not call object storage for canonical previous-session source resolution.
- Required canonical previous-session input missing:
- fails with guidance to run `narratio run-stage --force prepare`.
- Optional missing sources are omitted from adapter input paths.
## Skip and Resume Behavior
## Skip and resume behavior
- Runner-level skip applies when analyze is already `succeeded` and `--force` is not set.
- Analyze remains stage-scoped for resume/skip; there is no per-artifact resume state.
- `--artifacts` filters which configured artifacts are executable when analyze runs; it does not imply `--force`.
- Analyze is stage-scoped for resume; no per-artifact manifest resume state.
- `--artifacts` filters executable artifacts but does not imply force rerun.
## Failure Behavior
- Fails on invalid dependency ordering, unavailable required configured inputs, invalid built-in input prerequisites, render/run adapter failures, validation-failed adapter results, or missing/empty outputs.
- Required configured dependency missing from catalog availability fails clearly before invocation.
- Optional missing inputs are omitted.
## Failure behavior
- Fails on dependency-order violations, missing required inputs, resolver validation failures, adapter errors, and missing/empty generated outputs.
- Required unavailable configured artifact source (`narratio.artifact.<name>`) fails before invocation.
- Required canonical previous-session source fails with prepare-rerun guidance.
## Tests to Inspect Before Changing
## Tests to inspect before changing
- `internal/stage/analyze_test.go`
- `internal/artifacts/catalog_test.go`
- `internal/artifacts/artifact_resolver_test.go`
- `internal/adapters/scriptorium/subprocess_test.go`
- `internal/app/restore_workflow_test.go`
## Architectural Invariants
- Configured artifacts are identified by `narratio.artifact.<name>` source IDs.
- Artifact-to-artifact references rely on explicit `depends_on` declarations validated in config.
- Generated analyze outputs are treated uniformly as Scriptorium artifacts.
- Successful outputs must exist and be non-empty before promotion.
## Architectural invariants
- Canonical previous-session behavior is local-cache only during analyze.
- Generated outputs are validated and promoted before stage success is recorded.
- Resolver/catalog decisions stay deterministic and validation-gated.

View File

@@ -1,17 +1,20 @@
# Stage: archive
## Purpose
Publish run records and promoted session artifacts to object storage, then atomically advance the remote current pointer.
Publish durable run/session state to object storage, then atomically advance remote current state.
## Inputs and Outputs
Inputs:
- session manifest and prerequisite stage records
- run root contents under `runs/{run_id}/`
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
- effective source-based promotion locks from static config and remote session lock store
- session-level `previous/**` cache files when present
Outputs:
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
- uploaded promoted artifacts under `{session_prefix}/...`
- uploaded session previous-cache files under `{session_prefix}/previous/...` when present
- `{session_prefix}/current/manifest.json`
- `{session_prefix}/current/run_id.txt` written last
@@ -21,6 +24,8 @@ Owns:
- Prerequisite stage success enforcement
- Run file collection and upload (excluding `audio/`)
- Promotion rule resolution and upload
- Promotion lock enforcement
- Session previous-cache file collection/upload
- Commit pointer publish order
Does not own:
@@ -31,6 +36,8 @@ Does not own:
- `pipeline.archive.enabled`
- `pipeline.archive.upload_run`
- `pipeline.archive.promote_artifacts`
- `pipeline.archive.locks`
- `{session_prefix}/locks.yml` loaded by app orchestration before archive execution
- `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix`
- `pipeline.workspace.root`
@@ -43,8 +50,13 @@ Does not own:
## State and Manifest Behavior
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
- Resolves bucket/prefix from manifest identity first, then config fallback.
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
- Skips top-level promotion uploads for effective locked sources; run-local uploads still publish.
- Effective locks are the union of `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
- Writes metadata including:
- upload counts/paths
- `previous_files_uploaded` and `previous_uploaded_paths`
- `locked_promotion_count` and `locked_promotions`
- `current_manifest_key`
- `current_run_id_key`
- `current_pointer_written`
@@ -55,7 +67,8 @@ Does not own:
- Runner-level skip also applies for previously succeeded stage unless forced.
## Failure Behavior
- Fails on missing prerequisite success, missing object store when required, missing run root, missing required promotion source, upload failures, or pointer write failures.
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
- Locked required promotions are intentional skips and do not fail archive.
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
## Tests to Inspect Before Changing
@@ -64,5 +77,8 @@ Does not own:
## Architectural Invariants
- Run upload excludes `audio/` subtree.
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
- Ordinary `--force` does not override archive locks.
- Malformed or unreadable remote lock store fails archive-capable execution before promotion.
- `current/manifest.json` uploads before `current/run_id.txt`.
- `current/run_id.txt` is the remote publish commit marker.

View File

@@ -1,40 +1,51 @@
# Stage: prepare
## Purpose
Materialize all required session inputs into canonical local workspace paths and record input provenance in the session manifest.
Materialize canonical current-session input state and provenance before downstream stages run.
## Inputs and Outputs
Prepare owns:
- local input file materialization (`inputs/**`);
- audio input materialization (`audio/**`);
- previous-session cache hydration (`previous/**`) for canonical previous-session artifact sources.
## Inputs and outputs
Inputs:
- `session.yml` (resolved session config)
- `pipeline.resolved.yml` (materialized from resolved pipeline config)
- `speakers.yml`
- `autocorrect.yml`
- `glossary.yml`
- resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
- remote session provenance when `session.yml` was loaded from S3;
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
- audio source:
- local (`session.inputs.audio_dir` or `session.inputs.audio_files`), or
- S3 (`session.inputs.audio_s3.prefix`)
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
- S3: `session.inputs.audio_s3.prefix`;
- configured enabled Scriptorium artifact inputs (for previous-session requirement scanning);
- remote previous-session current archive state when previous hydration is required.
Outputs:
- `inputs/session.yml`
- `inputs/pipeline.resolved.yml`
- `inputs/speakers.yml`
- `inputs/autocorrect.yml`
- `inputs/glossary.yml`
- `audio/*.flac` in session workdir
- `manifest.Inputs` records with checksums and source metadata
- `inputs/campaign.yml`;
- `inputs/session.yml`;
- `inputs/pipeline.resolved.yml`;
- `inputs/speakers.yml`;
- `inputs/autocorrect.yml`;
- `inputs/glossary.yml`;
- `audio/*.flac` in canonical session `audio/`;
- optional `previous/manifest.json`;
- optional `previous/artifacts/**`;
- deterministic `manifest.Inputs` records with checksums and provenance metadata.
## Boundaries
Owns:
- Input path resolution and validation
- Local copy/materialization of configs and audio files
- S3 audio download to run-scoped spool, then copy into work audio dir
- input path resolution and materialization;
- S3 audio list/download/copy flow;
- previous-session artifact requirement collection from enabled configured artifacts;
- previous cache lifecycle when requirements exist (clear and rehydrate managed `previous/` state).
Does not own:
- Transcript generation/processing
- Archive publish behavior
- transcript or artifact generation;
- analyze-stage source resolution;
- archive commit behavior.
## Config Fields Used
## Config fields used
- `session.session_id`
- `session.previous_session_id`
- `session.campaign`
- `session.inputs.speakers_file`
- `session.inputs.autocorrect_file`
@@ -44,31 +55,70 @@ Does not own:
- `session.inputs.audio_s3.prefix`
- `pipeline.workspace.root`
- `pipeline.spool.root`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix`
- `pipeline.scriptorium.artifacts.<name>.enabled`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
- `campaign.campaign`
- `campaign.inputs.speakers_file`
- `campaign.inputs.autocorrect_file`
- `campaign.inputs.glossary_file`
## External Adapters Used
- Object storage backend (`env.ObjectStore`) for S3 audio list/download when `audio_s3` is configured.
## External adapters used
- `storage.ObjectStore` for:
- S3 audio listing/downloads;
- previous-session current pointer/manifest/artifact object checks and downloads.
## State and Manifest Behavior
## State and manifest behavior
- Ensures workspace layout exists.
- Writes resolved config and input files to canonical `inputs/` paths.
- Records all prepared inputs into `manifest.Inputs` (sorted deterministically by kind/path).
- For S3 audio, records `S3Bucket`, `S3Key`, `S3Size`, `S3ETag`, and `SpoolPath` in each audio input record.
- Materializes canonical input files and audio files.
- For S3 audio, uses run-scoped spool for active downloads and durable cache for reusable audio files; cache hits copy directly to work audio without downloading the object again.
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
- Resolves session-provided stable input overrides relative to `session.yml`.
- Scans enabled configured artifact inputs for canonical sources:
- `narratio.previous_session.artifact.<artifact_key>`
- If one or more canonical previous-session requirements exist:
- clears managed `previous/` state;
- hydrates required/optional previous artifacts from the configured previous sessions committed archive current state;
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
- stores archive-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
## Skip and Resume Behavior
- Runner-level skip applies when stage already `succeeded` and `--force` is not set.
- Stage itself is deterministic/idempotent for unchanged inputs (`copyFileIfChanged`, `writeBytesIfChanged`).
## Required and optional previous-session behavior
- `previous_session_id` unset:
- if any referenced previous artifact is required: fail;
- if all referenced previous artifacts are optional: continue and omit them.
- Previous session archive current pointer or manifest missing:
- if any referenced previous artifact is required: fail;
- if all referenced previous artifacts are optional: continue and omit missing ones.
- Missing required previous artifact object: fail.
- Missing optional previous artifact object: omit.
- Downloaded previous artifacts must validate as non-empty files.
## Failure Behavior
- Fails on missing required files, invalid audio source combinations, no discoverable `.flac` files, duplicate audio basenames, missing object store for S3 mode, or S3 list/download failures.
## Skip and resume behavior
- Runner-level skip remains authoritative:
- if `prepare` already succeeded and run is not forced, `prepare` does not run and no hydration/download occurs.
- If `prepare` runs (including with `--force`), it owns managed `previous/` state for canonical previous-session inputs.
## Tests to Inspect Before Changing
## Failure behavior
- Fails on missing required input files, invalid audio-source combinations, empty/duplicate audio inputs, missing object store for S3 modes, and remote access/download/validation errors.
- For required canonical previous-session inputs, analyze-time missing-input guidance is to rerun:
- `narratio run-stage --force prepare`
## Tests to inspect before changing
- `internal/stage/prepare_test.go`
- `internal/app/session_cli_test.go`
- `internal/config/load_validate_test.go`
- `internal/stage/prepare_previous_test.go`
- `internal/artifacts/previous_requirements_test.go`
- `internal/app/runner_test.go`
## Architectural Invariants
## Architectural invariants
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
- Audio files must be `.flac`.
- Canonical `inputs/*` and `audio/*` paths are the durable source for downstream stages.
- Storage keys are computed by callers using archive/path helpers; storage adapter receives explicit keys.
- `prepare` is the only stage that hydrates canonical previous-session cache state.

View File

@@ -6,7 +6,8 @@ Document Narratio's remote storage backend contracts and implementations under `
## Inputs and outputs
Inputs:
- Resolved storage config (`pipeline.storage.*`).
- Bucket-relative object keys and local file paths from stage/app orchestration.
- Already-loaded environment variables for configured S3 credentials.
- Bucket-relative object keys and local file paths from app/stage orchestration.
Outputs:
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
@@ -22,6 +23,7 @@ Does not own:
- Session/run prefix semantics.
- Archive commit order semantics.
- Manifest updates.
- Filesystem secret loading from `pipeline.secrets.env_dir`.
## Config fields used
- `pipeline.storage.backend`
@@ -55,8 +57,10 @@ Implementations:
## Failure behavior
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
- App command orchestration loads configured filesystem secrets before calling the object-store factory.
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
## Tests to inspect before changing
- `internal/adapters/storage/factory_test.go`

View File

@@ -17,7 +17,8 @@ Outputs:
## Boundaries
Owns:
- Session-level path layout (`inputs/`, `audio/`, `transcripts/`, `artifacts/`, `reports/`, `logs/`, `config/`, `current/`, `runs/`)
- Session-level path layout (`inputs/`, `audio/`, `transcripts/`, `artifacts/`, `reports/`, `logs/`, `config/`, `current/`, `runs/`, `previous/`)
- `previous/manifest.json` and `previous/artifacts/**` are reserved for prepared previous-session state
- Run-local stage sandbox layout under `runs/{run_id}/{stage}/`
- Session lock acquisition/release (`.lock`)
@@ -31,6 +32,8 @@ Does not own:
- `pipeline.workspace.cleanup_after_archive`
- `pipeline.spool.root`
- `pipeline.spool.delete_audio_after_archive`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `session.campaign`
- `session.session_id`
@@ -42,7 +45,11 @@ None directly in this subsystem. Stages may use object storage adapters and then
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
- For S3 audio sessions, `prepare` records spool/work paths and S3 provenance in `manifest.Inputs`.
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
- `narratio clean --session-id <id>` removes the session work root and session spool root.
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
- `narratio clean --clear-cache` is the explicit opt-in for deleting matching S3 audio cache entries.
## Skip and Resume Behavior
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
@@ -65,4 +72,6 @@ None directly in this subsystem. Stages may use object storage adapters and then
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
- Run roots are always nested: `runs/{run_id}` under the session root.
- Run-local output promotion must end in canonical session paths.
- Cleanup only targets run-scoped directories and must never delete configured root directories.
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
- Automatic post-archive cleanup only targets run-scoped directories and must never delete configured root directories.
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.

View File

@@ -6,21 +6,37 @@ For field-level configuration, see [docs/config.md](./config.md). For full comma
## Normal workflow (S3-first path)
1. Upload session `.flac` files to object storage under the configured session audio prefix.
2. Run Narratio:
1. Create or upload `session.yml`, or pass a local `session.yml` explicitly.
2. Upload session `.flac` files to object storage under the configured session audio prefix.
3. Run Narratio:
```bash
narratio run --session-id 2026-04-04
```
3. Read success output:
4. Read success output:
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
- use `manifest=<path>` with `status` for inspection.
Notes:
- default config/session discovery applies unless `--config` and `--session` are passed.
- default config/campaign/session discovery checks system config locations unless `--config`, `--campaign`, and `--session` are passed.
- when local `session.yml` discovery misses, `--session-id` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
Initialize a remote session skeleton:
```bash
narratio session init --config /etc/narratio/pipeline.yml --campaign /etc/narratio/campaign.yml --session-id 2026-04-04 --remote
```
Remote init writes `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`. It fails if the object already exists unless `--force` is passed.
Validate before running:
```bash
narratio session validate --session-id 2026-04-04
```
## Restore workflow
Use restore when local durable session state is missing or stale and archive current state is authoritative.
@@ -40,7 +56,7 @@ narratio restore --session-id 2026-04-04
Post-restore analyze rerun pattern:
```bash
narratio run-stage --session-id 2026-04-04 --force analyze
narratio analyze --session-id 2026-04-04
```
Restore source-of-truth:
@@ -48,10 +64,20 @@ Restore source-of-truth:
- remote current manifest: `current/manifest.json`
Restore default scope:
- includes `manifest.json`, `transcripts/**`, `artifacts/**`
- includes `manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`
- includes `audio/**` only with `--include-audio`
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
Reset local state before restore testing:
```bash
narratio clean --session-id 2026-04-04 --dry-run
narratio clean --session-id 2026-04-04
narratio restore --session-id 2026-04-04 --include-audio
```
`clean` removes the local session work directory and session spool directory. It preserves the durable S3 audio cache by default, so repeated restore or forced prepare tests do not re-download large audio files.
## Local filesystem layout and state artifacts
Session root:
@@ -61,12 +87,14 @@ Primary state:
- `manifest.json`: session-level stage state.
- `runs/{run_id}/manifest.json`: invocation-level state.
- `.lock`: session lock while a modifying command is active.
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
Canonical session directories:
- `inputs/`
- `audio/`
- `transcripts/`
- `artifacts/`
- `previous/`
- `reports/`
- `logs/`
- `config/`
@@ -96,8 +124,15 @@ Configured artifact source reuse:
- reused configured artifact provenance is `filesystem.disabled_artifact_output`.
`--artifacts` behavior:
- accepted on `run`, `resume`, and `run-stage analyze`.
- filters analyze execution only; does not force stage rerun.
- accepted on `run`, `resume`, `run-stage analyze`, and `analyze`.
- filters analyze execution only.
- does not imply force on `run`, `resume`, or `run-stage`; `narratio analyze` is force-by-design.
Canonical previous-session input behavior:
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
- these inputs are hydrated by `prepare`, not `analyze`.
- if analyze fails due to missing canonical previous cache, rerun:
- `narratio run-stage --session-id <id> --force prepare`
## Remote archive layout and publish contract
@@ -109,6 +144,7 @@ When archive is enabled and run upload is enabled, archive publishes under:
Archive uploads:
- run record files from run root (excluding `audio/`).
- promoted files from explicit `archive.promote_artifacts` rules.
- mutable session locks from helper commands live at `{session_prefix}/locks.yml`.
Publish order:
1. upload `current/manifest.json`
@@ -122,6 +158,17 @@ Archive promotion is explicit and source-based:
- missing required promotion sources fail archive stage.
- missing optional promotion sources are skipped.
- invalid resolved artifacts fail archive stage.
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
- remote locks from `{session_prefix}/locks.yml` are merged with static `archive.locks`; static locks win on duplicate sources.
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
Lock helper behavior:
- `narratio locks --session-id <id>` lists effective static and remote locks.
- `narratio locks add --session-id <id> --reason <text> <source>` writes a remote lock.
- `narratio locks add --session-id <id> --force --reason <text> <source>` updates an existing remote lock reason.
- `narratio locks remove --session-id <id> <source>` removes only a remote lock.
- `locks remove` cannot remove static pipeline locks.
- remote lock writes check whether the lock store exists, but are not compare-and-swap atomic.
## Resume, retry, restore, and safe rerun behavior
@@ -139,6 +186,7 @@ Restore conflict policy:
Forced reruns:
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
- ordinary `--force` does not override archive locks.
Safe rerun pattern:
1. rerun the changed stage with `--force`.
@@ -146,12 +194,20 @@ Safe rerun pattern:
## Cleanup behavior
Cleanup is considered only when archive stage executed and succeeded.
Automatic post-archive cleanup is considered only when archive stage executed and succeeded.
Cleanup toggles:
Automatic cleanup toggles:
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
Manual cleanup:
- `narratio clean --session-id <id>` deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
- `narratio clean --all` deletes all local session work under `{workspace.root}/work` and all spool children under `{spool.root}`.
- `--dry-run` prints targets without deleting.
- `--clear-cache` also removes matching S3 audio cache files. Without it, cache is preserved.
The S3 audio cache under `pipeline.cache.root` is durable input cache state, not workspace or spool state. Automatic cleanup and default manual cleanup do not delete it.
Cleanup eligibility gates:
- archive enabled
- archive run upload enabled
@@ -178,18 +234,26 @@ Recommended recovery:
1. inspect state:
```bash
narratio status --session-id 2026-04-04
```
This reports local manifest state, committed remote current state, expected remote transcript/artifact availability, and archive locks.
2. for one manifest file, run:
```bash
narratio status --manifest <manifest-path>
```
2. for restore-specific checks, run:
3. for restore-specific checks, run:
```bash
narratio restore --session-id 2026-04-04 --dry-run
```
3. fix root cause (config/input/credentials/storage/service availability).
4. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
4. fix root cause (config/input/credentials/storage/service availability).
5. continue with `resume`, or targeted `run-stage --force` followed by `resume`.
## Restore report
@@ -206,7 +270,9 @@ Dry-run does not write restore report files.
## Operational caveats
- `status` requires explicit `--manifest`; there is no session-id lookup command.
- `status` with no config/session flags still requires explicit `--manifest`.
- `status --session-id <id>` uses normal config/session loading, including remote session fallback.
- `status --session-id <id>` includes the same promoted remote output availability view as `artifacts list --remote` when storage is configured.
- local and S3 audio input modes are mutually exclusive.
- archive publish requires upstream stages through `analyze` to be `succeeded`.
- required promotion rules can fail when selected analyze artifacts did not generate a required file path.

View File

@@ -0,0 +1,52 @@
# Roadmap: Operator Helper Commands
## Status
Implemented.
The operator helper command set is no longer conceptual. Current behavior is documented in:
- `docs/cli.md`
- `docs/operations.md`
- `docs/config.md`
- `docs/internal/artifacts.md`
- `docs/internal/stage-archive.md`
## Implemented Commands
- `narratio session validate`
- `narratio status --manifest <path>`
- `narratio status --session-id <id>`
- `narratio session init --output <path>`
- `narratio session init --remote`
- `narratio artifacts list`
- `narratio artifacts list --remote`
- `narratio locks`
- `narratio locks add <source>`
- `narratio locks remove <source>`
## Implemented Decisions
- Helper output is text-only. No JSON schema exists yet.
- `status` remains a top-level command.
- `session validate`, `session init`, and `artifacts list` are nested helper commands.
- `locks` is the single top-level command for listing, adding, and removing archive promotion locks.
- Remote session initialization requires explicit `--remote`.
- Local session initialization requires `--output`.
- Remote artifact availability is opt-in with `artifacts list --remote`.
- Mutable locks are source-based and stored at `{session_prefix}/locks.yml`.
- The remote lock store uses strict YAML with top-level `locks`.
- Static `pipeline.archive.locks` and remote locks are merged; static locks win on duplicate sources.
- `locks remove` removes only remote locks.
- Ordinary execution `--force` does not override locks.
- Remote lock writes use existence checks and `--force` for updates; there is no compare-and-swap protection.
## Remaining Future Enhancements
These are intentionally not implemented:
- `--json` output for helper commands.
- Optimistic concurrency or ETag compare-and-swap for remote lock mutations.
- Rich remote artifact availability across historical run-local objects.
- Session-lock acquisition for remote mutation helpers.
- Broader campaign helper commands such as `campaign validate` or `campaign publish`.

View File

@@ -1,715 +0,0 @@
# Roadmap: `narratio restore` Subcommand
## Status
Implemented through Step 8. This document remains as roadmap and design history for the restore feature, and as the home for future restore-related ideas (for example `run --restore`).
## Summary
Add a new `narratio restore` subcommand that hydrates a local session workspace from the current committed remote archive state.
The primary operator workflow is:
```bash
narratio restore --session-id 2026-04-04
narratio run-stage --force analyze
```
This should allow a new machine with no local workspace state to restore the durable session manifest, transcripts, and generated artifacts from S3, then generate new Scriptorium artifacts without re-running transcription, merge, normalize, polish, or trim.
This is intentionally a separate command. Do not fold this behavior into the `prepare` stage. The existing `prepare` stage should remain focused on materializing configured local/S3 inputs for a pipeline run.
## Goals
- Add a first-class `narratio restore` command.
- Restore the current committed remote session state into the canonical local session workspace.
- Use the existing object storage adapter boundary.
- Preserve archive commit semantics: only restore from a remote state that has a valid current commit marker.
- Restore durable session-level outputs needed for downstream stages, especially `analyze`.
- Provide safe conflict behavior by default.
- Support `--dry-run`, `--force`, and `--include-audio`.
- Keep the implementation explicit, testable, and narrow.
## Non-goals
- Do not make `restore` a pipeline stage.
- Do not change the `prepare` stage behavior as part of this work.
- Do not add implicit restore behavior to `narratio run` in this implementation.
- Do not restore historical run-local sandboxes by default.
- Do not implement a generic remote synchronization engine.
- Do not implement bidirectional sync.
- Do not delete local files merely because they are absent remotely.
- Do not merge remote and local manifests in the first implementation.
- Do not require live S3 for the ordinary unit test suite.
## Future work explicitly out of scope
A future change may add:
```bash
narratio run --restore
```
That future flag should run `narratio restore` before starting the normal pipeline. Mention this as future work in roadmap/docs if useful, but do not implement it now.
## Existing architecture to preserve
### `prepare` remains input materialization
The `prepare` stage currently materializes required session inputs into canonical local workspace paths and records input provenance. It owns local copying/materialization of config and audio inputs, including S3 audio download when `session.inputs.audio_s3.prefix` is configured. It does not own transcript generation/processing or archive publish behavior.
`restore` should not be implemented by expanding `prepare`. It should be an app-level command that reuses shared helpers where appropriate.
### Workspace model
The local durable session workspace is campaign-aware:
```text
{workspace.root}/work/{campaign}/{session_id}/
```
It contains durable session paths such as:
```text
manifest.json
inputs/
audio/
transcripts/
artifacts/
reports/
logs/
config/
current/
runs/
```
Run-local sandboxes live below:
```text
runs/{run_id}/
```
Restore should target durable session-level paths, not old run-local stage sandboxes.
### Storage boundary
The storage adapter owns object-store primitives only: `List`, `Download`, `Upload`, and `Exists`.
The storage adapter must not infer root prefixes, campaign names, session IDs, run IDs, or archive layout. Restore code must construct full bucket-relative keys before calling storage.
### Archive commit boundary
A remote run is current only after the archive stage has uploaded the run record, promoted outputs, `current/manifest.json`, and finally `current/run_id.txt`.
`current/run_id.txt` is the final remote commit marker and must be written last.
Restore must not treat incomplete, skipped, failed, or uncommitted archive attempts as current remote state.
## User-facing command
Add:
```bash
narratio restore [flags]
```
The command should use the same configuration/session discovery conventions as `run`, `plan`, `resume`, and `run-stage` where practical:
```bash
narratio restore --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-04-04
```
Required effective inputs:
- resolved pipeline config;
- resolved session config;
- `session.campaign`;
- `session.session_id`;
- configured remote storage backend.
Supported flags:
```text
--config <path> Existing pipeline config path behavior.
--session <path> Existing session config path behavior.
--session-id <value> Existing session template behavior.
--dry-run Plan restore actions without writing local files.
--force Overwrite conflicting local files with remote files.
--include-audio Include archived session-level audio files.
```
Do not add `--restore` to `run` in this implementation.
## Default restore scope
By default, restore:
1. Validates and reads the current remote commit marker.
2. Downloads the current remote manifest into the local session manifest path.
3. Downloads durable transcript files.
4. Downloads durable generated artifact files.
Default included remote/local durable paths:
```text
manifest.json from remote current manifest
transcripts/**
artifacts/**
```
Default excluded paths:
```text
audio/** unless --include-audio is passed
runs/** always excluded for this implementation
logs/** excluded for this implementation
reports/** excluded for this implementation unless needed for current manifest validation
config/** excluded for this implementation
inputs/** excluded for this implementation
current/** remote control metadata only; do not mirror blindly
```
If the existing archive implementation stores promoted files in a different remote layout, use the existing archive/path helpers and current archive semantics rather than inventing a parallel layout.
## Remote state discovery
Implement restore around the current committed archive state.
Expected algorithm:
1. Resolve pipeline/session config.
2. Ensure storage is configured.
3. Ensure local workspace layout exists.
4. Acquire the session lock.
5. Build the remote session archive prefix using the same helpers/policy used by archive code.
6. Check for the remote `current/run_id.txt` commit marker.
7. Read the committed run ID.
8. Download `current/manifest.json` to a temporary file.
9. Validate that the manifest is parseable and belongs to the requested campaign/session.
10. Build a restore plan from the committed remote state.
11. Execute the restore plan unless `--dry-run` is set.
12. Emit a concise summary.
Important: `current/run_id.txt` is the commit marker. Do not restore from a remote session prefix merely because files exist under `transcripts/` or `artifacts/`.
## Restore planning
Create a planning layer before writing files.
A restore plan entry should include at least:
```go
type RestoreAction struct {
Kind RestoreActionKind
RemoteKey string
LocalPath string
Size int64
ETag string
ExistsLocal bool
SameLocal bool
Conflict bool
Reason string
}
```
Suggested action kinds:
```text
download
skip_same
skip_missing_optional
conflict
```
The restore planner should be deterministic:
- sort remote objects by key;
- sort planned actions by local path or stable restore priority;
- write/report stable output for tests.
## Conflict and overwrite policy
Default behavior should be safe.
For each planned file:
```text
local absent:
download
local present and same as remote:
skip
local present and different:
conflict; fail restore unless --force is set
--force:
overwrite local conflicting files with remote versions
--dry-run:
do not write any files; report what would happen
```
The first implementation may use size and checksum/hash comparison where available. If remote ETag cannot be treated as a content hash, compare by downloading to a temporary file and hashing locally before deciding whether a local file is the same. Prefer correctness over assuming provider-specific ETag semantics.
Do not delete local files that are not present remotely.
## File writing and transactionality
Restore should avoid partial writes.
Implementation requirements:
- download each remote object to a temporary file under the session workspace or OS temp dir;
- validate downloaded content where possible before replacing local files;
- create parent directories as needed;
- atomically rename/copy into place only after successful download;
- do not overwrite local files unless `--force` is set;
- if a later file fails, preserve already-restored files but return a failure summary;
- never corrupt an existing local manifest on failed manifest download/parse.
Manifest restore is especially sensitive:
- download remote `current/manifest.json` to a temporary file;
- parse and validate it;
- if no local manifest exists, install it;
- if a local manifest exists and is equivalent, skip;
- if a local manifest exists and differs, fail unless `--force` is set;
- with `--force`, replace the local manifest with the remote manifest after validation;
- do not attempt a manifest merge in the initial implementation.
## Manifest semantics
`restore` is not a pipeline run and should not mark stages as running/succeeded/failed.
The restored remote manifest becomes the local session manifest. That is what allows a subsequent command such as:
```bash
narratio run-stage --force analyze
```
to see existing upstream stage state and canonical durable outputs.
Do not create a new run manifest for `restore`.
It is acceptable to write a restore diagnostic report outside the manifest, for example:
```text
reports/restore-latest.json
```
or a timestamped report, if that pattern fits the existing codebase. The report must not contain secrets.
## Local workspace locking
`restore` should acquire the same session lock used by ordinary pipeline operations before modifying session workspace state.
If the lock is held, fail fast with the same lock-conflict behavior used elsewhere.
`--dry-run` may still acquire the lock for consistency, but it is acceptable to avoid the lock if the codebase already has a clear read-only command pattern. Prefer safety and simplicity.
## Audio behavior
By default, do not restore audio.
If `--include-audio` is passed:
- restore archived durable session-level audio files only;
- do not use run-scoped spool paths;
- do not mutate or delete spool state;
- do not infer original `session.inputs.audio_s3.prefix` behavior;
- respect the same conflict/force/dry-run behavior used for transcripts/artifacts.
If the archive does not contain durable audio files, `--include-audio` should report that no archived audio was found rather than failing, unless the final implementation chooses to treat explicit audio restore as required. Prefer non-failure for absent archived audio unless tests or existing archive semantics suggest otherwise.
## Remote object selection
Prefer using manifest/artifact metadata when it reliably identifies durable outputs.
Also support listing committed durable archive prefixes so restore can retrieve all top-level session artifacts that may not yet be fully represented in manifest metadata.
The implementation should inspect existing archive code before choosing the final object-selection method. Do not duplicate archive path construction.
Recommended selection priority:
1. Remote current manifest path.
2. Durable promoted transcript/artifact outputs recorded in the manifest or archive metadata, if available.
3. Objects under committed durable `transcripts/` and `artifacts/` archive prefixes.
4. Objects under durable `audio/` only when `--include-audio` is passed.
Always exclude:
```text
runs/**
```
for the first implementation.
## Package and file organization
Expected areas to inspect and update:
```text
cmd/narratio/
internal/app/
internal/adapters/storage/
internal/artifacts/
internal/manifest/
docs/
examples/
```
Suggested implementation shape:
```text
internal/app/restore.go
internal/app/restore_test.go
internal/archive/restore/
planner.go
executor.go
report.go
keys.go
*_test.go
```
The exact package name may vary. Use whatever best fits the existing repository, but keep these boundaries clear:
- `internal/app` owns CLI command handling, config/session loading, lock acquisition, and wiring.
- Restore planning/execution owns remote key discovery, conflict detection, downloads, and reporting.
- `internal/adapters/storage` remains a transport boundary only.
- Workspace/path helpers remain centralized; do not scatter string concatenation.
If the repository already has an `internal/archive` or archive-stage helper package, prefer extending that rather than creating a conflicting package layout.
## CLI output
`narratio restore` should print a concise operator summary.
Example successful output:
```text
Restored session archive for sample-campaign/2026-04-04
Remote run: 20260504T031500Z-a1b2c3
Downloaded: 4
Skipped unchanged: 2
Conflicts: 0
```
Example dry run:
```text
Restore plan for sample-campaign/2026-04-04
Remote run: 20260504T031500Z-a1b2c3
Would download: transcripts/processed.json
Would download: transcripts/trimmed.json
Would skip unchanged: artifacts/session_recap.md
```
Example conflict:
```text
restore conflict: local artifacts/session_recap.md differs from remote archive; rerun with --force to overwrite
```
Do not print transcript or artifact content.
## Error behavior
Fail clearly when:
- storage backend is not configured;
- S3 bucket/config is missing or invalid;
- remote current commit marker is missing;
- remote current manifest is missing;
- remote manifest is invalid;
- remote manifest does not match requested campaign/session;
- local file differs from remote and `--force` is not set;
- a required remote object download fails;
- a local path would escape the session workspace;
- a remote key maps to an unsafe local path.
Skip or report non-fatal conditions when:
- optional audio restore finds no archived audio;
- an included prefix has no objects;
- a local file already matches the remote file.
## Path safety
Every restored file must map to a safe path under the session root.
Validation rules:
- local restore paths must be relative to the session root;
- reject absolute paths;
- reject `..` traversal;
- reject paths that escape through symlinks if the codebase has symlink-safe path checks;
- do not restore remote keys directly without mapping/classification;
- do not mirror arbitrary remote keys.
## Testing plan
Add focused unit tests. Do not require live S3.
### CLI tests
Add or update `internal/app` command tests for:
- `narratio restore --help`;
- restore accepts `--config`, `--session`, and `--session-id`;
- restore accepts `--dry-run`;
- restore accepts `--force`;
- restore accepts `--include-audio`;
- restore fails when storage is not configured;
- restore does not run pipeline stages.
### Restore planner tests
Test:
- missing `current/run_id.txt` fails;
- missing `current/manifest.json` fails;
- invalid manifest fails;
- wrong campaign/session manifest fails;
- default scope includes manifest/transcripts/artifacts;
- default scope excludes audio/logs/reports/config/runs;
- `--include-audio` includes durable audio;
- run-local keys are excluded;
- keys are sorted deterministically;
- unsafe remote-to-local paths are rejected.
### Conflict policy tests
Test:
- absent local file downloads;
- matching local file skips;
- differing local file conflicts by default;
- `--force` overwrites conflicts;
- `--dry-run` writes nothing;
- partial failure does not corrupt an existing local manifest.
### Storage/fake tests
Use fake storage to simulate:
- object listing;
- object download;
- missing objects;
- download failures;
- metadata/ETag behavior.
### Workspace/lock tests
Test:
- session layout is created before restore;
- session lock conflict fails;
- restored files land under the expected campaign/session workspace;
- no files are written outside the session root.
### Follow-up command workflow test
Add at least one test that simulates:
```bash
narratio restore --session-id 2026-04-04
narratio run-stage --force analyze
```
The test does not need to run real Scriptorium. Use existing fake/stub behavior to verify that restored transcripts and manifest state are sufficient for analyze-stage input resolution.
## Documentation updates when implemented
When the feature is implemented, update current-behavior docs:
```text
docs/cli.md
docs/operations.md
docs/internal/storage.md or docs/internal/archive/restore.md
```
If the documentation set does not yet have an internal restore document, add one consistent with the existing internal-doc style:
```text
docs/internal/command-restore.md
```
or:
```text
docs/internal/archive-restore.md
```
Do not document future `narratio run --restore` behavior outside `docs/roadmap/` until implemented.
## Implementation phases
### Phase 1: Audit existing archive and path helpers (completed)
Before coding behavior, inspect:
```text
internal/app/
internal/stage/archive*
internal/adapters/storage/
internal/artifacts/
internal/manifest/
docs/internal/stage-archive.md, if present
```
Determine:
- exact remote archive key layout;
- how root prefix/campaign/session are modeled;
- how current commit marker keys are built;
- how current manifest is uploaded;
- where promoted outputs are uploaded;
- whether helper functions already exist for remote archive keys;
- whether local workspace path helpers can safely map restore destinations.
Deliverable:
- small code comments or internal helper selection;
- no large behavior change yet unless required by tests.
### Phase 2: Add CLI surface and command wiring (completed)
Add `narratio restore` command parsing.
Wire flags:
```text
--config
--session
--session-id
--dry-run
--force
--include-audio
```
Use the existing config/session load path where practical.
Deliverable:
- command exists;
- help output is sensible;
- command validates basic inputs;
- command returns a clear “not yet implemented” or calls an empty planner if phased commits are desired;
- CLI tests pass.
### Phase 3: Implement remote current-state discovery (completed)
Add restore code that:
- creates an object store from resolved config;
- builds remote current marker key;
- reads `current/run_id.txt`;
- reads/downloads `current/manifest.json`;
- validates manifest identity;
- returns remote current-state metadata.
Deliverable:
- fake-storage tests for current-state discovery;
- no local file writes beyond temporary files.
### Phase 4: Implement restore planning (completed)
Build deterministic restore plans for default scope and `--include-audio`.
Deliverable:
- plan lists manifest, transcript, artifact files;
- plan excludes run-local data;
- plan detects local same/conflict/missing states;
- dry-run output works;
- no real file overwrite yet except temp comparisons as needed.
### Phase 5: Implement restore execution (completed)
Execute the plan safely:
- create directories;
- download to temporary files;
- validate content where practical;
- atomically install files;
- enforce default conflict failure;
- support `--force`;
- preserve existing manifest unless safe to replace.
Deliverable:
- restore works end-to-end against fake storage;
- failures are clear and do not corrupt existing local manifest.
### Phase 6: Add restore report and operator summary (completed)
Add concise stdout summary and optional JSON restore report if consistent with project diagnostics.
Deliverable:
- user-friendly output;
- durable diagnostic report if implemented;
- no content leakage.
### Phase 7: Workflow integration test (completed)
Add a test for restoring a previous session and then forcing `analyze`.
Deliverable:
- restored manifest/transcripts/artifacts are sufficient for analyze input resolution;
- no upstream stages rerun;
- no reliance on live subprocesses or S3.
### Phase 8: Documentation update (completed)
Once implemented, update current-behavior docs and internal command docs.
Also leave future `narratio run --restore` in roadmap only.
## Definition of done
The feature is complete when:
- `narratio restore` exists and is documented.
- It uses the same config/session discovery semantics as other commands where practical.
- It requires configured remote storage.
- It restores only from a committed current archive state.
- It restores the current manifest, transcripts, and artifacts by default.
- It restores audio only with `--include-audio`.
- It excludes run-local sandboxes.
- It fails on local/remote conflicts by default.
- `--force` overwrites conflicts.
- `--dry-run` writes nothing.
- It uses fake storage in tests.
- It does not change `prepare` behavior.
- It does not implement `narratio run --restore`.
- It avoids AWS SDK leakage outside the storage adapter.
- It uses centralized path/key helpers rather than scattered string concatenation.
- `go test ./...` passes.
## Suggested test commands
Run focused tests first:
```bash
go test ./internal/app -run TestExecute -v
go test ./internal/adapters/storage -v
go test ./internal/artifacts -v
go test ./internal/manifest -v
```
Then run the full suite:
```bash
go test ./...
```
## Suggested commit message
```text
Add restore subcommand roadmap
```

View File

@@ -9,19 +9,19 @@ Symptom:
- `run`, `plan`, `resume`, `run-stage`, or `restore` fails with config/session not found.
Likely Cause:
- `pipeline.yml` or `session.yml` is missing from discovery paths.
- wrong working directory when relying on `./session.yml`.
- `pipeline.yml`, `campaign.yml`, or `session.yml` is missing from system discovery paths.
- a local working-directory config file was not passed explicitly.
Diagnostics:
```bash
pwd
ls -l ./session.yml
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
ls -l /usr/local/etc/narratio/campaign.yml /etc/narratio/campaign.yml
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
```
Safe Fix:
- pass explicit `--config` and `--session`.
- pass explicit `--config`, `--campaign`, and `--session`.
- or place files in documented discovery paths.
Links:
@@ -40,7 +40,7 @@ Likely Cause:
Diagnostics:
```bash
narratio plan --session ./session.yml --session-id 2026-04-04
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session ./session.yml --session-id 2026-04-04
```
Safe Fix:
@@ -62,7 +62,7 @@ Likely Cause:
Diagnostics:
```bash
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
```
Safe Fix:
@@ -84,7 +84,7 @@ Likely Cause:
Diagnostics:
```bash
narratio run --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
narratio run --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout
```
Safe Fix:
@@ -106,7 +106,7 @@ Likely Cause:
Diagnostics:
```bash
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts session_recap polish
```
Safe Fix:
@@ -129,7 +129,7 @@ Likely Cause:
Diagnostics:
```bash
narratio plan --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04
narratio plan --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04
```
Safe Fix:
@@ -154,7 +154,7 @@ Diagnostics:
```bash
narratio status --manifest /path/to/manifest.json
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --artifacts player_handout analyze
```
Safe Fix:
@@ -227,7 +227,7 @@ Likely Cause:
Diagnostics:
```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
```
Safe Fix:
@@ -250,7 +250,7 @@ Likely Cause:
Diagnostics:
```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
```
Safe Fix:
@@ -272,7 +272,7 @@ Likely Cause:
Diagnostics:
```bash
narratio restore --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
narratio restore --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 --dry-run
```
Safe Fix:
@@ -342,7 +342,7 @@ Likely Cause:
Diagnostics:
```bash
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 prepare
```
Safe Fix:
@@ -366,7 +366,7 @@ Diagnostics:
```bash
narratio status --manifest /path/to/manifest.json
narratio run-stage --config /path/to/pipeline.yml --session /path/to/session.yml --session-id 2026-04-04 archive
narratio run-stage --config /path/to/pipeline.yml --campaign /path/to/campaign.yml --session /path/to/session.yml --session-id 2026-04-04 archive
```
Safe Fix:

5
examples/campaign.yml Normal file
View File

@@ -0,0 +1,5 @@
campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

View File

@@ -133,9 +133,7 @@ scriptorium:
source: narratio.transcript.trimmed
required: true
previous_recap:
source: previous_session_artifact
artifact: session_recap
path: ""
source: narratio.previous_session.artifact.session_recap
required: false
vars:
session_id: true

View File

@@ -81,8 +81,7 @@ scriptorium:
source: narratio.transcript.trimmed
required: true
previous_recap:
source: previous_session_artifact
artifact: session_recap
source: narratio.previous_session.artifact.session_recap
required: false
vars:
session_id: true

View File

@@ -1,9 +1,5 @@
session_id: 2026-05-03
campaign: sample-campaign
date: 2026-05-03
title: Sample Session
inputs:
audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -1,10 +1,6 @@
session_id: 2026-05-03
campaign: sample-campaign
date: 2026-05-03
title: Sample Session
inputs:
audio_s3:
prefix: audio/
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -1,7 +1,3 @@
session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./examples/speakers.yml
autocorrect_file: ./examples/autocorrect.yml
glossary_file: ./examples/glossary.yml

View File

@@ -27,8 +27,9 @@ type FakeBackend struct {
Err error
Result ArchiveResult
Objects map[string]FakeObject
Uploads []FakeUploadCall
Objects map[string]FakeObject
Uploads []FakeUploadCall
Downloads []FakeDownloadCall
ListErr error
DownloadErr error
@@ -43,6 +44,12 @@ type FakeUploadCall struct {
Options UploadOptions
}
// FakeDownloadCall captures one download invocation in call order.
type FakeDownloadCall struct {
Key string
LocalPath string
}
// Archive records request and returns configured response.
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
if err := ctx.Err(); err != nil {
@@ -130,6 +137,10 @@ func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error
if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
}
f.Downloads = append(f.Downloads, FakeDownloadCall{
Key: normalizeObjectKey(key),
LocalPath: localPath,
})
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", key, err)

View File

@@ -56,7 +56,7 @@ func TestFakeBackendDownload(t *testing.T) {
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
t.Fatalf("Download() error = %v", err)
}
data, err := os.ReadFile(dst)

View File

@@ -9,17 +9,19 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
[]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
&stdout,
&stderr,
)
@@ -33,12 +35,12 @@ func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"run", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
[]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
&stdout,
&stderr,
)
@@ -52,7 +54,7 @@ func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -65,7 +67,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
var out bytes.Buffer
err := RunStage(
context.Background(),
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
&out,
)
if err != nil {
@@ -78,7 +80,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -93,7 +95,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
var out bytes.Buffer
err := Resume(
context.Background(),
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap"},
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
&out,
)
if err != nil {
@@ -104,10 +106,154 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
}
}
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string) {
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var capturedStages []string
var capturedForce bool
origExecuteStagesFn := executeStagesFn
t.Cleanup(func() {
executeStagesFn = origExecuteStagesFn
})
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
for _, s := range stages {
capturedStages = append(capturedStages, s.Name())
}
capturedForce = opts.Force
return &RunSummary{
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
Executed: []string{"analyze"},
}, nil
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath},
&stdout,
&stderr,
)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
}
if !capturedForce {
t.Fatal("captured force = false, want true")
}
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
}
}
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var capturedArtifacts []string
origExecuteStagesFn := executeStagesFn
t.Cleanup(func() {
executeStagesFn = origExecuteStagesFn
})
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{
"analyze",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--artifacts", "player_handout,session_recap",
},
&stdout,
&stderr,
)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
}
}
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"analyze", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
&stdout,
&stderr,
)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
}
}
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
cases := []struct {
name string
args []string
want string
}{
{name: "positional", args: []string{"analyze", "extra"}, want: "analyze: unexpected positional arguments"},
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(tc.args, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), tc.want) {
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
}
})
}
}
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"analyze"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
}
}
func TestExecuteUsageIncludesAnalyze(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(nil, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "analyze") {
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
}
}
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
if err != nil {
t.Fatalf("open pipeline config for append: %v", err)
@@ -136,5 +282,5 @@ scriptorium:
if _, err := f.WriteString(extra); err != nil {
t.Fatalf("append scriptorium config: %v", err)
}
return pipelinePath, sessionPath
return pipelinePath, campaignPath, sessionPath
}

View File

@@ -0,0 +1,49 @@
package app
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func resolveCampaignConfigPath(flagValue string) (string, error) {
return resolveCampaignConfigPathWithCandidates(flagValue, config.DefaultCampaignConfigSearchPaths)
}
func resolveCampaignConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
if explicit := strings.TrimSpace(flagValue); explicit != "" {
return explicit, nil
}
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
if path == "" {
continue
}
ordered = append(ordered, path)
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default campaign config %q: %w", path, err)
}
if len(ordered) == 0 {
return "", fmt.Errorf("no campaign config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no campaign config path provided and no default campaign config found; searched: %s; pass --campaign to use an explicit path",
strings.Join(ordered, ", "),
)
}

View File

@@ -0,0 +1,49 @@
package app
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveCampaignConfigPathExplicitWins(t *testing.T) {
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
got, err := resolveCampaignConfigPathWithCandidates(explicit, []string{filepath.Join(t.TempDir(), "campaign.yml")})
if err != nil {
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
}
if got != explicit {
t.Fatalf("path = %q, want explicit path %q", got, explicit)
}
}
func TestResolveCampaignConfigPathUsesFirstExistingDefault(t *testing.T) {
dir := t.TempDir()
missing := filepath.Join(dir, "missing.yml")
found := filepath.Join(dir, "campaign.yml")
if err := os.WriteFile(found, []byte("campaign: sample-campaign\n"), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
got, err := resolveCampaignConfigPathWithCandidates("", []string{missing, found})
if err != nil {
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(found) {
t.Fatalf("path = %q, want %q", got, filepath.Clean(found))
}
}
func TestResolveCampaignConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveCampaignConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/campaign.yml", "/etc/narratio/campaign.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "searched") {
t.Fatalf("error = %q, want searched paths", err.Error())
}
if !strings.Contains(err.Error(), "pass --campaign") {
t.Fatalf("error = %q, want explicit-campaign guidance", err.Error())
}
}

336
internal/app/clean.go Normal file
View File

@@ -0,0 +1,336 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// Clean removes local workspace/spool state while preserving durable cache
// state unless cache cleanup is explicitly requested.
func Clean(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var all bool
var dryRun bool
var clearCache bool
addCommonConfigFlags(fs, &flags)
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("clean: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("clean: unexpected positional arguments")
}
if all {
return cleanAllLocal(flags, dryRun, clearCache, out)
}
return cleanSession(ctx, flags, dryRun, clearCache, out)
}
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("clean: --session-id is required unless --all is set")
}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return fmt.Errorf("clean: %w", err)
}
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return fmt.Errorf("clean: resolved pipeline and session config are required")
}
campaign := strings.TrimSpace(cfg.Session.Campaign)
sessionID := strings.TrimSpace(cfg.Session.SessionID)
if campaign == "" || sessionID == "" {
return fmt.Errorf("clean: campaign and session_id are required")
}
if dryRun {
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
} else {
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
}
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if clearCache {
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
return fmt.Errorf("clean: %w", err)
}
} else {
fmt.Fprintln(out, "Cache: preserved")
}
return nil
}
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
if strings.TrimSpace(flags.campaignPath) != "" ||
strings.TrimSpace(flags.sessionPath) != "" ||
strings.TrimSpace(flags.sessionID) != "" ||
strings.TrimSpace(flags.previousSessionID) != "" {
return fmt.Errorf("clean: --all cannot be combined with --campaign, --session, --session-id, or --previous-session-id")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
if err != nil {
return fmt.Errorf("clean: %w", err)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
if err != nil {
return fmt.Errorf("clean: %w", err)
}
if dryRun {
fmt.Fprintln(out, "Clean plan for all local sessions")
} else {
fmt.Fprintln(out, "Cleaned all local sessions")
}
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
return fmt.Errorf("clean: %w", err)
}
if clearCache {
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
return fmt.Errorf("clean: %w", err)
}
} else {
fmt.Fprintln(out, "Cache: preserved")
}
return nil
}
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
dir, err := validateScopedDir(root, target, policy)
if err != nil {
return err
}
if dryRun {
if dir.Exists {
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
} else {
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
}
return nil
}
if !dir.Exists {
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
return nil
}
if err := os.RemoveAll(dir.TargetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
}
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
return nil
}
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
rootAbs, entries, err := cleanableRootChildren(root, policy)
if err != nil {
return err
}
if len(entries) == 0 {
if dryRun {
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
} else {
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
}
return nil
}
for _, entry := range entries {
if dryRun {
fmt.Fprintf(out, "Would delete: %s\n", entry)
continue
}
if err := os.RemoveAll(entry); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
}
fmt.Fprintf(out, "Deleted: %s\n", entry)
}
return nil
}
func cleanableRootChildren(root, policy string) (string, []string, error) {
cleanRoot := strings.TrimSpace(root)
if cleanRoot == "" {
return "", nil, fmt.Errorf("cleanup policy %s: root path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
info, err := os.Lstat(rootAbs)
if err != nil {
if os.IsNotExist(err) {
return rootAbs, nil, nil
}
return "", nil, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
}
if !info.IsDir() {
return "", nil, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
}
entries, err := os.ReadDir(rootAbs)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
}
out := make([]string, 0, len(entries))
for _, entry := range entries {
path := filepath.Join(rootAbs, entry.Name())
info, err := os.Lstat(path)
if err != nil {
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
}
out = append(out, path)
}
return rootAbs, out, nil
}
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
if cfg.Session.Inputs.AudioS3 == nil {
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
return nil
}
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
objects, err := store.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
count := 0
for _, obj := range objects {
key := strings.TrimSpace(obj.Key)
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
continue
}
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
if err != nil {
return err
}
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
if err != nil {
return err
}
if deleted {
count++
}
}
if count == 0 {
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
}
return nil
}
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
}
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
if err != nil {
return err
}
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
}
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
file, err := validateScopedFile(root, target, policy)
if err != nil {
return false, err
}
if dryRun {
if file.Exists {
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
return true, nil
}
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
return false, nil
}
if !file.Exists {
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
return false, nil
}
if err := os.Remove(file.TargetAbs); err != nil {
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
}
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
return true, nil
}
func validateScopedFile(root, target, policy string) (scopedDir, error) {
cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" {
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
}
if cleanTarget == "" {
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
targetAbs, err := filepath.Abs(cleanTarget)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
}
rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil {
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
}
if rel == "." {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
}
info, err := os.Lstat(targetAbs)
if err != nil {
if os.IsNotExist(err) {
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
}
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
}
if info.IsDir() {
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
}
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
}
func cleanIsFlac(path string) bool {
return strings.EqualFold(filepath.Ext(path), ".flac")
}

255
internal/app/clean_test.go Normal file
View File

@@ -0,0 +1,255 @@
package app
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
mustWriteTestFile(t, cachePath, "cached-audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, workDir)
cleanAssertMissing(t, spoolDir)
cleanAssertExists(t, cachePath)
if !strings.Contains(stdout.String(), "Cache: preserved") {
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
}
}
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--dry-run"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertExists(t, workDir)
cleanAssertExists(t, spoolDir)
if !strings.Contains(stdout.String(), "Would delete:") {
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
}
}
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "Missing:") {
t.Fatalf("stdout = %q, want missing path output", stdout.String())
}
}
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
inputs:
audio_s3:
prefix: audio/
`), 0o644); err != nil {
t.Fatalf("write session: %v", err)
}
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
cacheRoot := filepath.Join(workspaceRoot, "cache")
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, cachePath, "cached-audio")
mustWriteTestFile(t, otherCachePath, "other-audio")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, cachePath)
cleanAssertExists(t, otherCachePath)
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
}
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
}
}
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
workRoot := filepath.Join(workspaceRoot, "work")
spoolRoot := filepath.Join(workspaceRoot, "spool")
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
mustWriteTestFile(t, cachePath, "cache")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, workRoot)
cleanAssertExists(t, spoolRoot)
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
cleanAssertExists(t, cachePath)
}
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
cacheRoot := filepath.Join(workspaceRoot, "cache")
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
mustWriteTestFile(t, audioCachePath, "cached-audio")
mustWriteTestFile(t, otherCachePath, "other-cache")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
cleanAssertMissing(t, audioCachePath)
cleanAssertExists(t, otherCachePath)
}
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean", "--config", pipelinePath, "--campaign", campaignPath, "--all"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "--all cannot be combined") {
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
}
}
func TestCleanRequiresSessionID(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clean"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "--session-id is required unless --all is set") {
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
}
}
func TestCleanRejectsUnsafeTargets(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
t.Fatal("outside target error = nil, want error")
}
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
t.Fatal("root target error = nil, want error")
}
filePath := filepath.Join(root, "file.txt")
mustWriteTestFile(t, filePath, "file")
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
t.Fatal("file target error = nil, want error")
}
symlinkPath := filepath.Join(root, "link")
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
t.Fatalf("Symlink() error = %v", err)
}
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
t.Fatal("symlink target error = nil, want error")
}
}
func TestClearIsNotCommandAlias(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"clear"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
}
}
func cleanAssertExists(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected %q to exist: %v", path, err)
}
}
func cleanAssertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
}
}

View File

@@ -7,7 +7,7 @@ import (
"strings"
)
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "restore"}
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage", "analyze", "restore", "session", "artifacts", "locks", "clean"}
// Execute dispatches CLI commands and returns a process exit code.
func Execute(args []string, stdout, stderr io.Writer) int {
@@ -32,8 +32,18 @@ func Execute(args []string, stdout, stderr io.Writer) int {
err = Resume(ctx, cmdArgs, stdout)
case "run-stage":
err = RunStage(ctx, cmdArgs, stdout)
case "analyze":
err = Analyze(ctx, cmdArgs, stdout)
case "restore":
err = Restore(ctx, cmdArgs, stdout)
case "session":
err = Session(ctx, cmdArgs, stdout)
case "artifacts":
err = Artifacts(ctx, cmdArgs, stdout)
case "locks":
err = Locks(ctx, cmdArgs, stdout)
case "clean":
err = Clean(ctx, cmdArgs, stdout)
default:
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
printUsage(stderr)

View File

@@ -24,7 +24,7 @@ func TestExecuteValidCommands(t *testing.T) {
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
manifestPath := writeManifestPathForExecute(t)
cases := []struct {
@@ -32,11 +32,11 @@ func TestExecuteValidCommands(t *testing.T) {
args []string
wantOut string
}{
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
{name: "run", args: []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
}
for _, tc := range cases {
@@ -94,12 +94,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
func TestExecuteRunStageUnknownFails(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -110,14 +110,14 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -136,19 +136,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
code = Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -188,6 +188,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
root: ` + workspaceRoot + `
@@ -239,7 +240,7 @@ inputs:
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -252,6 +253,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
@@ -288,7 +290,7 @@ inputs:
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -305,11 +307,14 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
config.DefaultCampaignConfigSearchPaths = []string{campaignPath}
defer func() {
config.DefaultPipelineConfigSearchPaths = originalDefaults
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
}()
var stdout bytes.Buffer
@@ -323,6 +328,33 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
}
}
func TestExecuteMissingCampaignConfigReportsSearchedPaths(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
missingCampaignPath := filepath.Join(t.TempDir(), "campaign.yml")
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
config.DefaultCampaignConfigSearchPaths = []string{missingCampaignPath}
defer func() {
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
}()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if !strings.Contains(stderr.String(), "no campaign config path provided and no default campaign config found; searched:") {
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
}
if !strings.Contains(stderr.String(), "pass --campaign") {
t.Fatalf("stderr = %q, want explicit campaign guidance", stderr.String())
}
}
func TestExecuteInvalidCommand(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -359,11 +391,12 @@ func TestExecuteMissingCommand(t *testing.T) {
}
}
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
t.Helper()
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml")
url := "https://example.com/transcribe"
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
@@ -377,6 +410,10 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
pipelineYAML := `workspace:
root: ` + workspaceRoot + `
cache:
root: ` + filepath.Join(workspaceRoot, "cache") + `
spool:
root: ` + filepath.Join(workspaceRoot, "spool") + `
storage:
backend: s3
s3:
@@ -407,9 +444,11 @@ notification:
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
`
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
@@ -418,6 +457,9 @@ inputs:
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline config: %v", err)
}
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign config: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session config: %v", err)
}
@@ -427,7 +469,22 @@ inputs:
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
return pipelinePath, sessionPath
return pipelinePath, campaignPath, sessionPath
}
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
t.Helper()
campaignPath := filepath.Join(dir, "campaign.yml")
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
return campaignPath
}
func writeManifestPathForExecute(t *testing.T) string {

View File

@@ -0,0 +1,133 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
if err != nil {
return nil, err
}
resolvedCampaignPath, err := resolveCampaignConfigPath(campaignFlag)
if err != nil {
return nil, err
}
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, explicitSession, sessionOpts)
}
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
if err != nil {
return nil, err
}
if discoveredSession.Path != "" {
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, discoveredSession.Path, sessionOpts)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
if err != nil {
return nil, err
}
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
if err != nil {
return nil, err
}
sessionID := strings.TrimSpace(sessionOpts.SessionID)
if sessionID == "" {
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires --session-id")
}
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
partialCfg := &config.Config{
Pipeline: pipelineCfg,
Campaign: campaignCfg,
PipelinePath: resolvedPipelinePath,
CampaignPath: resolvedCampaignPath,
}
store, err := newCommandObjectStore(ctx, partialCfg, nil)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
}
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
}
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
if err != nil {
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
}
sessionBytes, err := os.ReadFile(sessionTempPath)
if err != nil {
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
}
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(pipelineCfg)+"/"+remoteKey, sessionBytes, sessionOpts)
if err != nil {
return nil, err
}
return config.Resolve(
resolvedPipelinePath,
pipelineCfg,
resolvedCampaignPath,
campaignCfg,
sessionTempPath,
sessionCfg,
config.SessionSource{
Source: "session_config.s3",
LocalPath: sessionTempPath,
S3Bucket: s3BucketName(pipelineCfg),
S3Key: remoteKey,
S3Size: sessionInfo.Size,
S3ETag: sessionInfo.ETag,
SpoolPath: sessionTempPath,
},
)
}
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
objects, err := store.List(ctx, sessionPrefix)
if err != nil {
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
}
for _, obj := range objects {
if obj.Key == remoteKey {
return obj, nil
}
}
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
}
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
f, err := os.CreateTemp("", "narratio-session-*.yml")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := f.Name()
if err := f.Close(); err != nil {
return "", fmt.Errorf("close temp file %q: %w", path, err)
}
if err := store.Download(ctx, remoteKey, path); err != nil {
return "", err
}
return filepath.Clean(path), nil
}
func s3BucketName(cfg *config.PipelineConfig) string {
if cfg == nil || cfg.Storage.S3 == nil {
return ""
}
return strings.TrimSpace(cfg.Storage.S3.Bucket)
}

View File

@@ -0,0 +1,21 @@
package app
import (
"context"
"fmt"
"log/slog"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
return nil, fmt.Errorf("load secrets from files: %w", err)
}
store, err := newObjectStoreFromConfigFn(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
}
return store, nil
}

View File

@@ -0,0 +1,165 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
fake := &storage.FakeBackend{}
called := false
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
return nil, errors.New("access key was not loaded before object store init")
}
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
return nil, errors.New("secret key was not loaded before object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
store, err := newCommandObjectStore(context.Background(), cfg, nil)
if err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
if store != fake {
t.Fatalf("store = %#v, want fake backend", store)
}
if !called {
t.Fatal("object store factory was not called")
}
}
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
t.Setenv(accessKeyEnv, "existing-key-id")
t.Setenv(secretKeyEnv, "existing-secret")
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
return nil, errors.New("existing access key was overwritten")
}
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
return nil, errors.New("existing secret key was overwritten")
}
return &storage.FakeBackend{}, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
}
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
called := false
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
return &storage.FakeBackend{}, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
_, err := newCommandObjectStore(context.Background(), cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if called {
t.Fatal("object store factory was called after secret load failure")
}
if !strings.Contains(err.Error(), "load secrets from files") {
t.Fatalf("error = %q, want secret loading context", err.Error())
}
}
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
cfg := commandObjectStoreTestConfig("")
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
return nil, errors.New("factory boom")
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
_, err := newCommandObjectStore(context.Background(), cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
t.Fatalf("error = %q, want factory context", err.Error())
}
}
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{
Backend: "s3",
S3: &config.StorageS3Config{
Bucket: "test-bucket",
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
},
},
},
}
if strings.TrimSpace(secretsDir) != "" {
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
}
return cfg
}
func restoreEnvAfterTest(t *testing.T, names ...string) {
t.Helper()
originals := make(map[string]string, len(names))
present := make(map[string]bool, len(names))
for _, name := range names {
value, ok := os.LookupEnv(name)
originals[name] = value
present[name] = ok
_ = os.Unsetenv(name)
}
t.Cleanup(func() {
for _, name := range names {
if present[name] {
_ = os.Setenv(name, originals[name])
} else {
_ = os.Unsetenv(name)
}
}
})
}

View File

@@ -0,0 +1,958 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gopkg.in/yaml.v3"
)
type commonConfigFlags struct {
pipelinePath string
campaignPath string
sessionPath string
sessionID string
previousSessionID string
}
type finding struct {
Severity string
Category string
Message string
}
type findingError struct {
count int
}
func (e findingError) Error() string {
return fmt.Sprintf("%d validation error(s)", e.count)
}
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&flags.campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
}
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
return config.SessionLoadOptions{
SessionID: f.sessionID,
PreviousSessionID: f.previousSessionID,
}
}
// Session dispatches session helper subcommands.
func Session(ctx context.Context, args []string, out io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("session: expected subcommand: validate|init")
}
switch args[0] {
case "validate":
return SessionValidate(ctx, args[1:], out)
case "init":
return SessionInit(ctx, args[1:], out)
default:
return fmt.Errorf("session: unknown subcommand %q", args[0])
}
}
// Artifacts dispatches artifact helper subcommands.
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("artifacts: expected subcommand: list")
}
switch args[0] {
case "list":
return ArtifactsList(ctx, args[1:], out)
default:
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
}
}
// SessionValidate performs a read-only session preflight.
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("session validate: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("session validate: unexpected positional arguments")
}
findings := []finding{}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
findings = append(findings, errorFinding("config", err.Error()))
return renderFindings(out, "", "", findings)
}
if err := config.Validate(cfg); err != nil {
findings = append(findings, errorFinding("config", err.Error()))
} else {
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
}
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
findings = append(findings, validateStableInputFindings(cfg)...)
findings = append(findings, validateLocalAudioFindings(cfg)...)
store, storeErr := objectStoreIfConfigured(ctx, cfg)
if storeErr != nil {
findings = append(findings, errorFinding("storage", storeErr.Error()))
}
if cfg.Session.Inputs.AudioS3 != nil {
if storeErr != nil {
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
} else {
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
}
}
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
if len(requirements) == 0 {
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
} else if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
} else if storeErr != nil {
findings = append(findings, errorFinding("previous", "previous-session artifacts cannot be checked because storage is unavailable"))
} else {
findings = append(findings, validatePreviousArtifactFindings(ctx, cfg, store, requirements)...)
}
locks, lockErr := loadEffectiveLocks(ctx, cfg, store)
if lockErr != nil {
findings = append(findings, errorFinding("locks", lockErr.Error()))
} else if len(locks.All) == 0 {
findings = append(findings, okFinding("locks", "no effective archive locks"))
} else {
for _, lock := range locks.All {
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
}
}
if paths.ManifestPath != "" {
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
}
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
}
// Status reports either a requested manifest or effective local/remote session state.
func Status(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var manifestPath string
var flags commonConfigFlags
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("status: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("status: unexpected positional arguments")
}
if strings.TrimSpace(manifestPath) != "" {
return statusManifest(ctx, manifestPath, out)
}
if flags.pipelinePath == "" && flags.campaignPath == "" && flags.sessionPath == "" && flags.sessionID == "" && flags.previousSessionID == "" {
return fmt.Errorf("status: --manifest is required")
}
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return fmt.Errorf("status: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("status: %w", err)
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
} else if m == nil {
fmt.Fprintln(out, "Local manifest: missing")
} else {
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
writeStageStatuses(out, m)
}
store, storeErr := objectStoreIfConfigured(ctx, cfg)
if storeErr != nil {
fmt.Fprintf(out, "Remote archive: unavailable: %v\n", storeErr)
} else if store != nil {
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
if err != nil {
fmt.Fprintf(out, "Remote archive: missing or unavailable: %v\n", err)
} else {
fmt.Fprintf(out, "Remote archive: current run %s\n", current.RunID)
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
}
}
locks, err := loadEffectiveLocks(ctx, cfg, store)
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
} else if storeErr == nil {
catalogLocks := locks
if err != nil {
catalogLocks = &effectiveLocks{
Static: staticArchiveLocks(cfg),
All: staticArchiveLocks(cfg),
}
}
promotedRemoteState := map[string]string{}
if store != nil {
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
}
fmt.Fprintln(out, "Remote outputs:")
writeArtifactList(out, cfg, catalog, catalogLocks, promotedRemoteState)
}
if err != nil {
fmt.Fprintf(out, "Archive locks: error: %v\n", err)
} else {
writeLocks(out, cfg, locks)
}
fmt.Fprintln(out, "Next actions:")
fmt.Fprintf(out, "- narratio session validate --session-id %s\n", cfg.Session.SessionID)
fmt.Fprintf(out, "- narratio restore --session-id %s --dry-run\n", cfg.Session.SessionID)
return nil
}
func statusManifest(ctx context.Context, manifestPath string, out io.Writer) error {
store := &manifest.LocalStore{}
m, err := store.Load(ctx, manifestPath)
if err != nil {
return fmt.Errorf("status: %w", err)
}
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
return err
}
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
return err
}
writeStageStatuses(out, m)
return nil
}
// SessionInit creates a local or remote session.yml skeleton.
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var pipelinePath, campaignPath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
var remote, force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
fs.StringVar(&date, "date", "", "session date")
fs.StringVar(&title, "title", "", "session title")
fs.StringVar(&output, "output", "", "local output session.yml path")
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
fs.BoolVar(&force, "force", false, "overwrite existing target")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("session init: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("session init: unexpected positional arguments")
}
if strings.TrimSpace(pipelinePath) == "" || strings.TrimSpace(campaignPath) == "" || strings.TrimSpace(sessionID) == "" {
return fmt.Errorf("session init: --config, --campaign, and --session-id are required")
}
if (strings.TrimSpace(output) == "") == !remote {
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
}
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
}
resolvedPipeline, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
resolvedCampaign, err := resolveCampaignConfigPath(campaignPath)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
pipelineCfg, err := config.LoadPipeline(resolvedPipeline)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
campaignCfg, err := config.LoadCampaign(resolvedCampaign)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
data, err := buildSessionYAML(campaignCfg.Campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
label := strings.TrimSpace(output)
if label == "" {
label = "remote session.yml"
}
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("session init: %w", err)
}
cfg, err := config.Resolve(resolvedPipeline, pipelineCfg, resolvedCampaign, campaignCfg, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
if err != nil {
return fmt.Errorf("session init: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("session init: %w", err)
}
if !remote {
if err := writeLocalFile(output, data, force); err != nil {
return fmt.Errorf("session init: %w", err)
}
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
return err
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return fmt.Errorf("session init: %w", err)
}
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
key := artifacts.S3SessionConfigKey(sessionPrefix)
exists, err := store.Exists(ctx, key)
if err != nil {
return fmt.Errorf("session init: check remote session %q: %w", key, err)
}
if exists && !force {
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
}
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
if err != nil {
return fmt.Errorf("session init: create temp file: %w", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("session init: write temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("session init: close temp file: %w", err)
}
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
}
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(pipelineCfg), key)
return err
}
// ArtifactsList lists effective artifact sources.
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var remote bool
addCommonConfigFlags(fs, &flags)
fs.BoolVar(&remote, "remote", false, "inspect remote archive availability")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("artifacts list: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("artifacts list: unexpected positional arguments")
}
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
if err != nil {
return fmt.Errorf("artifacts list: %w", err)
}
catalog, err := buildHelperArtifactCatalog(cfg)
if err != nil {
return fmt.Errorf("artifacts list: %w", err)
}
promotedRemoteState := map[string]string{}
if remote && store != nil {
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
}
writeArtifactList(out, cfg, catalog, locks, promotedRemoteState)
return nil
}
// Locks dispatches archive lock list and mutation helpers.
func Locks(ctx context.Context, args []string, out io.Writer) error {
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
switch args[0] {
case "add":
return LocksAdd(ctx, args[1:], out)
case "remove":
return LocksRemove(ctx, args[1:], out)
default:
return fmt.Errorf("locks: unknown subcommand %q", args[0])
}
}
return LocksList(ctx, args, out)
}
// LocksList lists effective archive locks.
func LocksList(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("locks: unexpected positional arguments")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks: --session-id is required")
}
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks: %w", err)
}
writeLocks(out, cfg, locks)
return nil
}
// LocksAdd adds or updates one remote lock.
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
var reason string
var force bool
addCommonConfigFlags(fs, &flags)
fs.StringVar(&reason, "reason", "", "lock reason")
fs.BoolVar(&force, "force", false, "update existing remote lock")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks add: invalid flags: %w", err)
}
if fs.NArg() != 1 {
return fmt.Errorf("locks add: expected exactly one source id")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks add: --session-id is required")
}
source := strings.TrimSpace(fs.Arg(0))
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks add: %w", err)
}
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
return fmt.Errorf("locks add: %w", err)
}
if _, ok := lockSourceSet(locks.Static)[source]; ok {
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
}
remoteSet := lockSourceSet(locks.Remote)
if _, exists := remoteSet[source]; exists && !force {
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
}
remoteSet[source] = config.ArchiveLockRule{Source: source, Reason: strings.TrimSpace(reason)}
remoteLocks := lockMapValues(remoteSet)
if _, err := config.ValidateArchiveLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
return fmt.Errorf("locks add: %w", err)
}
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
return fmt.Errorf("locks add: %w", err)
}
_, err = fmt.Fprintf(out, "narratio locks add: locked %s\n", source)
return err
}
// LocksRemove removes one remote lock.
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var flags commonConfigFlags
addCommonConfigFlags(fs, &flags)
if err := fs.Parse(args); err != nil {
return fmt.Errorf("locks remove: invalid flags: %w", err)
}
if fs.NArg() != 1 {
return fmt.Errorf("locks remove: expected exactly one source id")
}
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("locks remove: --session-id is required")
}
source := strings.TrimSpace(fs.Arg(0))
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
if err != nil {
return fmt.Errorf("locks remove: %w", err)
}
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
return fmt.Errorf("locks remove: %w", err)
}
remoteSet := lockSourceSet(locks.Remote)
if _, ok := remoteSet[source]; !ok {
if _, static := lockSourceSet(locks.Static)[source]; static {
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
}
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
}
delete(remoteSet, source)
remoteLocks := lockMapValues(remoteSet)
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
return fmt.Errorf("locks remove: %w", err)
}
_, err = fmt.Fprintf(out, "narratio locks remove: unlocked %s\n", source)
return err
}
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.sessionPath, flags.sessionOptions())
if err != nil {
return nil, nil, nil, nil, err
}
if err := config.Validate(cfg); err != nil {
return nil, nil, nil, nil, err
}
var store storage.ObjectStore
if needStore {
store, err = newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return nil, nil, nil, nil, err
}
} else {
store, _ = objectStoreIfConfigured(ctx, cfg)
}
locks, err := loadEffectiveLocks(ctx, cfg, store)
if err != nil {
return nil, nil, nil, nil, err
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
m, err := loadLocalManifest(ctx, paths.ManifestPath)
if err != nil {
return nil, nil, nil, nil, err
}
return cfg, store, locks, m, nil
}
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
return nil, nil
}
store, err := newCommandObjectStore(ctx, cfg, nil)
if err != nil {
return nil, err
}
return store, nil
}
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
date = strings.TrimSpace(sessionID)
}
type audioS3 struct {
Prefix string `yaml:"prefix"`
}
type inputs struct {
AudioDir string `yaml:"audio_dir,omitempty"`
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
}
type sessionYAML struct {
Campaign string `yaml:"campaign"`
SessionID string `yaml:"session_id"`
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
Date string `yaml:"date,omitempty"`
Title string `yaml:"title,omitempty"`
Inputs inputs `yaml:"inputs"`
}
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
if in.AudioDir == "" {
prefix := strings.TrimSpace(audioS3Prefix)
if prefix == "" {
prefix = "audio/"
}
in.AudioS3 = &audioS3{Prefix: prefix}
}
data, err := yaml.Marshal(sessionYAML{
Campaign: strings.TrimSpace(campaign),
SessionID: strings.TrimSpace(sessionID),
PreviousSessionID: strings.TrimSpace(previousSessionID),
Date: strings.TrimSpace(date),
Title: strings.TrimSpace(title),
Inputs: in,
})
if err != nil {
return nil, err
}
return data, nil
}
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
if campaign != "" || sessionID != "" {
fmt.Fprintf(out, "Campaign: %s\n", campaign)
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
}
errorsCount := 0
for _, f := range findings {
if f.Severity == "ERROR" {
errorsCount++
}
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
}
if errorsCount > 0 {
return findingError{count: errorsCount}
}
return nil
}
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
func sessionSourceSummary(cfg *config.Config) string {
source := cfg.SessionSource.Source
if source == "" {
source = "session_config"
}
if cfg.SessionSource.S3Key != "" {
return source + " " + cfg.SessionSource.S3Key
}
return source + " " + cfg.SessionPath
}
func validateStableInputFindings(cfg *config.Config) []finding {
items := []struct {
name string
in config.ResolvedInputFile
}{
{"speakers", cfg.StableInputs.SpeakersFile},
{"autocorrect", cfg.StableInputs.AutocorrectFile},
{"glossary", cfg.StableInputs.GlossaryFile},
}
out := make([]finding, 0, len(items))
for _, item := range items {
path, err := resolveHelperConfigRelativePath(item.in)
if err != nil {
out = append(out, errorFinding("inputs", item.name+": "+err.Error()))
continue
}
if _, err := os.Stat(path); err != nil {
out = append(out, errorFinding("inputs", fmt.Sprintf("%s missing: %v", item.name, err)))
} else {
out = append(out, okFinding("inputs", item.name+": "+path))
}
}
return out
}
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
if strings.TrimSpace(input.ConfigPath) == "" {
return "", fmt.Errorf("source config path is required")
}
path := strings.TrimSpace(input.Path)
if path == "" {
return "", fmt.Errorf("path is required")
}
if filepath.IsAbs(path) {
return filepath.Clean(path), nil
}
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
}
func validateLocalAudioFindings(cfg *config.Config) []finding {
if cfg.Session.Inputs.AudioS3 != nil {
return nil
}
audioDir := strings.TrimSpace(cfg.Session.Inputs.AudioDir)
if audioDir == "" && len(cfg.Session.Inputs.AudioFiles) == 0 {
return []finding{errorFinding("audio", "audio_dir, audio_files, or audio_s3 is required")}
}
base := filepath.Dir(cfg.SessionPath)
paths := []string{}
if audioDir != "" {
dir := audioDir
if !filepath.IsAbs(dir) {
dir = filepath.Join(base, dir)
}
matches, err := filepath.Glob(filepath.Join(dir, "*.flac"))
if err != nil || len(matches) == 0 {
return []finding{errorFinding("audio", "no .flac files found in "+dir)}
}
paths = append(paths, matches...)
}
for _, file := range cfg.Session.Inputs.AudioFiles {
p := file
if !filepath.IsAbs(p) {
p = filepath.Join(base, p)
}
paths = append(paths, p)
}
for _, p := range paths {
if _, err := os.Stat(p); err != nil {
return []finding{errorFinding("audio", fmt.Sprintf("audio file missing: %v", err))}
}
}
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(paths)))}
}
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
objects, err := store.List(ctx, audioPrefix)
if err != nil {
return errorFinding("audio", err.Error())
}
count := 0
for _, obj := range objects {
if strings.HasSuffix(strings.ToLower(obj.Key), ".flac") {
count++
}
}
if count == 0 {
return errorFinding("audio", "no remote .flac objects found under "+audioPrefix)
}
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", count))
}
func validatePreviousArtifactFindings(ctx context.Context, cfg *config.Config, store storage.ObjectStore, requirements []artifacts.PreviousArtifactRequirement) []finding {
out := []finding{}
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(prefix)
for _, key := range []string{runIDKey, manifestKey} {
exists, err := store.Exists(ctx, key)
if err != nil {
out = append(out, errorFinding("previous", fmt.Sprintf("check %s: %v", key, err)))
return out
}
if !exists {
out = append(out, errorFinding("previous", "missing "+key))
return out
}
}
for _, req := range requirements {
out = append(out, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
}
return out
}
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
store := &manifest.LocalStore{}
return store.Load(ctx, path)
}
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
if m == nil || len(m.Stages) == 0 {
fmt.Fprintln(out, "stages: no stages recorded")
return
}
fmt.Fprintln(out, "stages:")
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
}
}
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
catalog := artifacts.NewArtifactCatalog()
if err := catalog.RegisterBuiltIns(); err != nil {
return nil, err
}
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
if cfg.Pipeline.Scriptorium != nil {
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
}
}
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
return nil, err
}
return catalog, nil
}
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, promotedRemoteState map[string]string) {
lockSet := lockSourceSet(locks.All)
fmt.Fprintln(out, "Built-in:")
for _, id := range []string{
artifacts.ArtifactTranscriptMerged,
artifacts.ArtifactTranscriptPolished,
artifacts.ArtifactTranscriptFull,
artifacts.ArtifactTranscriptTrimmed,
artifacts.ArtifactBoundsSession,
} {
writeArtifactLine(out, id, lockSet)
}
fmt.Fprintln(out, "Configured:")
for _, entry := range catalog.ListConfigured() {
writeArtifactLine(out, entry.SourceID, lockSet)
}
fmt.Fprintln(out, "Previous-session:")
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
}
fmt.Fprintln(out, "Promoted:")
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
writePromotedArtifactLine(out, rule, catalog, lockSet, promotedRemoteState)
}
}
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.ArchiveLockRule) {
parts := []string{source}
if _, ok := lockSet[source]; ok {
parts = append(parts, "locked")
}
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
}
func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.ArchiveLockRule, remoteState map[string]string) {
source := strings.TrimSpace(rule.Source)
parts := []string{source}
if _, ok := lockSet[source]; ok {
parts = append(parts, "locked")
}
dest, showDest, err := helperPromotionDest(rule, catalog)
if err != nil {
parts = append(parts, "remote=error")
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
return
}
if showDest {
parts = append(parts, "dest="+dest)
}
if state := remoteState[promotionRemoteStateKey(source, dest)]; state != "" {
parts = append(parts, state)
}
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
}
func remotePromotionAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
out := map[string]string{}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
source := strings.TrimSpace(rule.Source)
dest, _, err := helperPromotionDest(rule, catalog)
if err != nil {
out[promotionRemoteStateKey(source, "")] = "remote=error"
continue
}
key := artifacts.S3PromotedArtifactKey(sessionPrefix, dest)
if exists, err := store.Exists(ctx, key); err == nil && exists {
out[promotionRemoteStateKey(source, dest)] = "remote=promoted"
} else if err != nil {
out[promotionRemoteStateKey(source, dest)] = "remote=error"
} else {
out[promotionRemoteStateKey(source, dest)] = "remote=missing"
}
}
return out
}
func helperPromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
source := strings.TrimSpace(rule.Source)
dest := strings.TrimSpace(rule.Dest)
if dest == "" {
entry, ok := catalog.Lookup(source)
if !ok {
return "", false, fmt.Errorf("destination omitted and source is unknown")
}
dest = strings.TrimSpace(entry.CanonicalRelPath)
if dest == "" {
return "", false, fmt.Errorf("destination omitted and no canonical destination is available")
}
}
normalized, err := normalizeHelperArchiveRelativePath(dest)
if err != nil {
return "", false, err
}
entry, ok := catalog.Lookup(source)
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
return normalized, showDest, nil
}
func normalizeHelperArchiveRelativePath(rel string) (string, error) {
trimmed := strings.TrimSpace(rel)
if trimmed == "" {
return "", fmt.Errorf("relative path is required")
}
cleaned := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
if cleaned == "." || cleaned == "" {
return "", fmt.Errorf("relative path is required")
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
return "", fmt.Errorf("path must be a clean relative path")
}
return cleaned, nil
}
func promotionRemoteStateKey(source, dest string) string {
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
}
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
if locks == nil || len(locks.All) == 0 {
fmt.Fprintln(out, "Archive locks: none")
return
}
fmt.Fprintln(out, "Archive locks:")
promoted := map[string]config.ArchivePromotionRule{}
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Archive != nil {
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
promoted[strings.TrimSpace(rule.Source)] = rule
}
}
staticSet := lockSourceSet(locks.Static)
for _, lock := range locks.All {
origin := "remote"
if _, ok := staticSet[lock.Source]; ok {
origin = "pipeline"
}
promo := "not-promoted"
if _, ok := promoted[lock.Source]; ok {
promo = "promoted"
}
reason := strings.TrimSpace(lock.Reason)
if reason == "" {
reason = "(no reason)"
}
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
}
}
func lockMapValues(in map[string]config.ArchiveLockRule) []config.ArchiveLockRule {
keys := make([]string, 0, len(in))
for key := range in {
keys = append(keys, key)
}
sort.Strings(keys)
out := make([]config.ArchiveLockRule, 0, len(keys))
for _, key := range keys {
item := in[key]
item.Source = key
item.Reason = strings.TrimSpace(item.Reason)
out = append(out, item)
}
return out
}

View File

@@ -0,0 +1,589 @@
package app
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestExecuteSessionInitRemoteWritesCanonicalSessionConfig(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"session", "init",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session-id", "2026-06-07",
"--title", "The Black Cabin",
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionConfigKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-06-07"))
obj, ok := fake.Objects[key]
if !ok {
t.Fatalf("remote session key %q not uploaded; objects=%v", key, fake.Objects)
}
if !strings.Contains(string(obj.Data), `session_id: "2026-06-07"`) || !strings.Contains(string(obj.Data), "prefix: audio/") {
t.Fatalf("remote session data = %q", string(obj.Data))
}
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
}
func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
accessKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
inputs:
audio_s3:
prefix: audio/
`), 0o644); err != nil {
t.Fatalf("write session: %v", err)
}
fake := &storage.FakeBackend{}
audioKey := artifacts.S3PromotedArtifactKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
origStoreFn := newObjectStoreFromConfigFn
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if os.Getenv(accessKeyEnv) != "test-key-id" || os.Getenv(secretKeyEnv) != "test-secret" {
return nil, fmt.Errorf("secrets were not loaded before object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"session", "validate", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if !strings.Contains(stdout.String(), "OK audio") {
t.Fatalf("stdout = %q, want OK audio", stdout.String())
}
}
func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "manual edit",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
obj, ok := fake.Objects[key]
if !ok {
t.Fatalf("remote locks key %q not uploaded", key)
}
if !strings.Contains(string(obj.Data), "source: narratio.transcript.trimmed") || !strings.Contains(string(obj.Data), "reason: manual edit") {
t.Fatalf("lock store data = %q", string(obj.Data))
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks list exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "- narratio.transcript.trimmed origin=remote") {
t.Fatalf("stdout = %q, want remote lock", stdout.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "remove",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
}
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
if err != nil {
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
}
if len(store.Locks) != 0 {
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
}
if storeInitCalls != 3 {
t.Fatalf("object store init calls = %d, want 3", storeInitCalls)
}
}
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "first",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("initial locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "second",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("duplicate locks add exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "pass --force to update") {
t.Fatalf("stderr = %q, want force guidance", stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--reason", "second",
"--force",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("forced locks add exit code = %d, want 0; stderr=%q", code, stderr.String())
}
key := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
if !strings.Contains(string(fake.Objects[key].Data), "reason: second") {
t.Fatalf("lock store data = %q, want updated reason", string(fake.Objects[key].Data))
}
}
func TestExecuteLocksRequireSessionID(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{"list", []string{"locks"}, "locks: --session-id is required"},
{"add", []string{"locks", "add", "narratio.transcript.trimmed"}, "locks add: --session-id is required"},
{"remove", []string{"locks", "remove", "narratio.transcript.trimmed"}, "locks remove: --session-id is required"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(tt.args, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), tt.want) {
t.Fatalf("stderr = %q, want %q", stderr.String(), tt.want)
}
})
}
}
func TestExecuteLocksCannotModifyStaticLocks(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addStaticArchiveLockToPipelineConfig(t, pipelinePath, "narratio.transcript.trimmed")
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"locks", "add",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("locks add static lock exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "locked by pipeline config") {
t.Fatalf("stderr = %q, want static lock error", stderr.String())
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{
"locks", "remove",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"narratio.transcript.trimmed",
}, &stdout, &stderr)
if code == 0 {
t.Fatal("locks remove static lock exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "locked by pipeline config") {
t.Fatalf("stderr = %q, want static lock error", stderr.String())
}
}
func TestExecuteTopLevelLockAndUnlockAreRemoved(t *testing.T) {
tests := []string{"lock", "unlock"}
for _, cmd := range tests {
t.Run(cmd, func(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{cmd, "narratio.transcript.trimmed"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), `unknown command: "`+cmd+`"`) {
t.Fatalf("stderr = %q, want unknown command", stderr.String())
}
})
}
}
func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
`)
fake := &storage.FakeBackend{}
trimmedKey := artifacts.S3PromotedArtifactKey(
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
"transcripts/trimmed.json",
)
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"artifacts", "list",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "narratio.transcript.trimmed remote=promoted") {
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
}
}
func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.full
dest: transcripts/full.json
required: true
- source: narratio.bounds.session
dest: transcripts/bounds.json
required: true
`)
fake := &storage.FakeBackend{}
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"artifacts", "list",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--remote",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
for _, unwanted := range []string{
"narratio.transcript.full remote=missing",
"narratio.bounds.session remote=missing",
} {
if strings.Contains(out, unwanted) {
t.Fatalf("stdout = %q, did not want catalog remote marker %q", out, unwanted)
}
}
for _, want := range []string{
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
"narratio.bounds.session dest=transcripts/bounds.json remote=promoted",
} {
if !strings.Contains(out, want) {
t.Fatalf("stdout = %q, want %q", out, want)
}
}
}
func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
- source: narratio.transcript.full
dest: transcripts/full.json
required: true
`)
fake := &storage.FakeBackend{}
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
trimmedKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
fullKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json")
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: fullKey, Data: []byte(`{"segments":[]}`)})
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"status",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
for _, want := range []string{
"Remote outputs:",
"Built-in:",
"Configured:",
"Previous-session:",
"Promoted:",
"narratio.transcript.trimmed locked",
"narratio.transcript.trimmed locked remote=promoted",
"narratio.transcript.full dest=transcripts/full.json remote=promoted",
} {
if !strings.Contains(out, want) {
t.Fatalf("stdout = %q, want %q", out, want)
}
}
if strings.Contains(out, "narratio.transcript.merged remote=missing") {
t.Fatalf("stdout = %q, did not want catalog remote marker", out)
}
}
func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
addArchivePromotionsToPipeline(t, pipelinePath, `
promote_artifacts:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
required: true
`)
fake := &storage.FakeBackend{ExistsErr: fmt.Errorf("exists failed")}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{
"status",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
out := stdout.String()
if !strings.Contains(out, "Remote archive: missing or unavailable:") {
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
}
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.trimmed remote=error") {
t.Fatalf("stdout = %q, want remote output error state", out)
}
if !strings.Contains(out, "Archive locks: error:") {
t.Fatalf("stdout = %q, want archive locks error", out)
}
}
func TestExecuteArchiveLoadsRemoteLocks(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidArchiveConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
lockKey := artifacts.S3SessionLocksKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"))
fake.SeedObject(storage.FakeObject{Key: lockKey, Data: []byte("locks:\n - source: narratio.transcript.trimmed\n reason: remote review\n")})
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
// The archive stage only checks the manifest statuses and source files.
_ = stageName
}
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "trimmed.json"), `{"segments":[]}`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "archive"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
promotedKey := artifacts.S3PromotedArtifactKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/trimmed.json")
if _, ok := fake.Objects[promotedKey]; ok {
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
}
}
func addArchivePromotionsToPipeline(t *testing.T, pipelinePath, archiveYAML string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(string(data), " upload_run: false\n", " upload_run: false\n"+archiveYAML, 1)
if updated == string(data) {
t.Fatalf("pipeline %q did not contain archive upload_run marker", pipelinePath)
}
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}
func writeValidArchiveConfigFiles(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(string(data), "upload_run: false", "upload_run: true", 1)
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
ctx := context.Background()
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", nowUTC())
m.Campaign = "sample-campaign"
m.RunID = "20260521T160000Z-test"
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
m.MarkStageSucceeded(name, nowUTC(), nil)
}
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
if err := store.Save(ctx, path, m); err != nil {
t.Fatalf("save manifest: %v", err)
}
runManifestPath := artifacts.SessionRunManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, m.RunID)
if err := os.MkdirAll(filepath.Dir(runManifestPath), 0o755); err != nil {
t.Fatalf("mkdir run manifest: %v", err)
}
if err := os.WriteFile(runManifestPath, []byte("{}\n"), 0o644); err != nil {
t.Fatalf("write run manifest: %v", err)
}
return pipelinePath, campaignPath, sessionPath
}
func addStaticArchiveLockToPipelineConfig(t *testing.T, pipelinePath, source string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
updated := strings.Replace(
string(data),
"archive:\n enabled: true\n upload_run: false\n",
"archive:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
1,
)
if updated == string(data) {
t.Fatalf("archive section not found in pipeline config")
}
if err := os.WriteFile(pipelinePath, []byte(updated), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}

View File

@@ -20,12 +20,16 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -34,17 +38,9 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("plan: unexpected positional arguments")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("plan: %w", err)

View File

@@ -15,10 +15,10 @@ import (
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
args := []string{"--config", pipelinePath, "--session", sessionPath}
args := []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}
if err := Plan(context.Background(), args, &out); err != nil {
t.Fatalf("first Plan() error = %v", err)
@@ -62,7 +62,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
}
var out bytes.Buffer
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out); err != nil {
t.Fatalf("Plan() error = %v", err)
}
got := out.String()
@@ -93,6 +93,7 @@ func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, configDir)
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
@@ -128,7 +129,7 @@ inputs:
}
var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}

View File

@@ -153,53 +153,70 @@ func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool,
return true, ""
}
type scopedDir struct {
RootAbs string
TargetAbs string
Exists bool
}
func removeRunScopedDir(root, target, policy string) error {
dir, err := validateScopedDir(root, target, policy)
if err != nil {
return err
}
if !dir.Exists {
return nil
}
if err := os.RemoveAll(dir.TargetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
}
return nil
}
func validateScopedDir(root, target, policy string) (scopedDir, error) {
cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" {
return fmt.Errorf("cleanup policy %s: root path is required", policy)
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
}
if cleanTarget == "" {
return fmt.Errorf("cleanup policy %s: target path is required", policy)
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
targetAbs, err := filepath.Abs(cleanTarget)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
}
rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil {
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
}
if rel == "." {
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
}
info, err := os.Lstat(targetAbs)
if err != nil {
if os.IsNotExist(err) {
return nil
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
}
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
}
if !info.IsDir() {
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
}
if err := os.RemoveAll(targetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
}
return nil
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
}
func asString(v any) string {

View File

@@ -82,6 +82,7 @@ func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
assertExists(t, cfg.Pipeline.Workspace.Root)
assertExists(t, seed.otherRunDir)
assertExists(t, seed.previousCachePath)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.spoolAudioDir)
}
@@ -98,6 +99,7 @@ func TestPostArchiveCleanupBothPolicies(t *testing.T) {
assertMissing(t, seed.spoolAudioDir)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.otherRunDir)
assertExists(t, seed.previousCachePath)
}
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
@@ -256,11 +258,12 @@ func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
}
type cleanupSeed struct {
runWorkDir string
otherRunDir string
spoolAudioDir string
localSourceAudio string
sessionPrefix string
runWorkDir string
otherRunDir string
spoolAudioDir string
localSourceAudio string
previousCachePath string
sessionPrefix string
}
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
@@ -274,11 +277,18 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
previousCachePath := artifacts.SessionPreviousArtifactPathForCampaign(
cfg.Pipeline.Workspace.Root,
cfg.Session.Campaign,
cfg.Session.SessionID,
"session_recap.md",
)
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
mustWriteFile(t, previousCachePath, "# previous recap\n")
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
mustWriteFile(t, localSourceAudio, "source\n")
@@ -301,11 +311,12 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
}
return cfg, cleanupSeed{
runWorkDir: runWorkDir,
otherRunDir: otherRunDir,
spoolAudioDir: spoolAudioDir,
localSourceAudio: localSourceAudio,
sessionPrefix: seed.S3SessionPrefix,
runWorkDir: runWorkDir,
otherRunDir: otherRunDir,
spoolAudioDir: spoolAudioDir,
localSourceAudio: localSourceAudio,
previousCachePath: previousCachePath,
sessionPrefix: seed.S3SessionPrefix,
}
}

View File

@@ -0,0 +1,157 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type effectiveLocks struct {
Static []config.ArchiveLockRule
Remote []config.ArchiveLockRule
All []config.ArchiveLockRule
Key string
}
func remoteLocksKey(cfg *config.Config) (string, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return "", fmt.Errorf("resolved config is required")
}
if cfg.Pipeline.Storage.S3 == nil {
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
}
sessionPrefix := artifacts.S3SessionPrefix(
cfg.Pipeline.Storage.S3.RootPrefix,
cfg.Session.Campaign,
cfg.Session.SessionID,
)
return artifacts.S3SessionLocksKey(sessionPrefix), nil
}
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.ArchiveLockStore, string, error) {
key, err := remoteLocksKey(cfg)
if err != nil {
return nil, "", err
}
exists, err := store.Exists(ctx, key)
if err != nil {
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
}
if !exists {
return &config.ArchiveLockStore{}, key, nil
}
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
if err != nil {
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
}
defer func() { _ = os.Remove(tmp) }()
data, err := os.ReadFile(tmp)
if err != nil {
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
}
lockStore, err := config.LoadArchiveLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
if err != nil {
return nil, key, err
}
return lockStore, key, nil
}
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
staticLocks := staticArchiveLocks(cfg)
if store == nil {
return &effectiveLocks{
Static: staticLocks,
All: append([]config.ArchiveLockRule(nil), staticLocks...),
}, nil
}
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
if err != nil {
return nil, err
}
remoteLocks := append([]config.ArchiveLockRule(nil), lockStore.Locks...)
return &effectiveLocks{
Static: staticLocks,
Remote: remoteLocks,
All: config.MergeArchiveLockRules(staticLocks, remoteLocks),
Key: key,
}, nil
}
func staticArchiveLocks(cfg *config.Config) []config.ArchiveLockRule {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
return nil
}
return append([]config.ArchiveLockRule(nil), cfg.Pipeline.Archive.Locks...)
}
func applyEffectiveLocks(cfg *config.Config, locks []config.ArchiveLockRule) {
if cfg == nil || cfg.Pipeline == nil {
return
}
if cfg.Pipeline.Archive == nil {
cfg.Pipeline.Archive = &config.ArchiveConfig{}
}
cfg.Pipeline.Archive.Locks = append([]config.ArchiveLockRule(nil), locks...)
}
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.ArchiveLockStore) error {
data, err := config.MarshalArchiveLockStore(lockStore)
if err != nil {
return err
}
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
if err != nil {
return fmt.Errorf("create lock store temp file: %w", err)
}
tmpPath := tmp.Name()
defer func() { _ = os.Remove(tmpPath) }()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write lock store temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close lock store temp file: %w", err)
}
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
return fmt.Errorf("upload remote locks %q: %w", key, err)
}
return nil
}
func lockSourceSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
out := make(map[string]config.ArchiveLockRule, len(locks))
for _, lock := range locks {
source := strings.TrimSpace(lock.Source)
if source == "" {
continue
}
lock.Source = source
lock.Reason = strings.TrimSpace(lock.Reason)
out[source] = lock
}
return out
}
func writeLocalFile(path string, data []byte, force bool) error {
cleaned := filepath.Clean(strings.TrimSpace(path))
if cleaned == "" || cleaned == "." {
return fmt.Errorf("output path is required")
}
if !force {
if _, err := os.Stat(cleaned); err == nil {
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
} else if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("check output file %q: %w", cleaned, err)
}
}
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
return fmt.Errorf("create output directory: %w", err)
}
return os.WriteFile(cleaned, data, 0o644)
}

View File

@@ -0,0 +1,252 @@
package app
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
inputs:
audio_s3:
prefix: audio/
`)
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 1 {
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
}
if !strings.Contains(stdout.String(), "narratio plan: workdir prepared") {
t.Fatalf("stdout = %q, want plan output", stdout.String())
}
if _, ok := fake.Objects[remoteKey]; !ok {
t.Fatalf("remote session key %q was not seeded", remoteKey)
}
}
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
accessKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
fake := &storage.FakeBackend{}
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
inputs:
audio_s3:
prefix: audio/
`)
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
var storeInitCalls int
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
}
if !strings.Contains(stderr.String(), missingSessionPath) {
t.Fatalf("stderr = %q, want local searched path", stderr.String())
}
}
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
var storeInitCalls int
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "remote session loading requires --session-id") {
t.Fatalf("stderr = %q, want session-id guidance", stderr.String())
}
if storeInitCalls != 0 {
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
}
}
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
return nil, errors.New("storage unavailable")
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
}
}
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
var storeInitCalls int
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "strict decode failed") {
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
}
}
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
t.Helper()
origStoreFn := newObjectStoreFromConfigFn
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
if storeInitCalls != nil {
(*storeInitCalls)++
}
return fake, nil
}
t.Cleanup(func() {
newObjectStoreFromConfigFn = origStoreFn
config.DefaultSessionConfigSearchPaths = origSessionDefaults
})
}
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
t.Helper()
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
fake.SeedObject(storage.FakeObject{
Key: remoteKey,
Data: []byte(content),
ETag: "remote-session-etag",
})
return remoteKey
}
func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv string) {
t.Helper()
pipelineData, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatalf("read pipeline: %v", err)
}
pipelineYAML := strings.Replace(
string(pipelineData),
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
1,
)
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline: %v", err)
}
}

View File

@@ -26,19 +26,23 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(out)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var dryRun bool
var force bool
var includeAudio bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.BoolVar(&dryRun, "dry-run", false, "plan restore actions without writing local files")
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
fs.Usage = func() {
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--session <path>] [--session-id <value>] [--dry-run] [--force] [--include-audio]")
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--campaign <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
_, _ = fmt.Fprintln(out)
_, _ = fmt.Fprintln(out, "Flags:")
fs.PrintDefaults()
@@ -53,17 +57,9 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("restore: unexpected positional arguments")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("restore: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("restore: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("restore: %w", err)
@@ -71,11 +67,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("restore: %w", err)
}
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
return fmt.Errorf("restore: %w", err)
}
objectStore, err := newObjectStoreFromConfigFn(ctx, cfg)
objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
if err != nil {
return fmt.Errorf("restore: %w", err)
}

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/audio"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -92,6 +93,10 @@ func executeRestoreDownloadAction(
return fmt.Errorf("restore plan local path mismatch for %q", action.LocalRelativePath)
}
if restoreActionIsAudio(action) {
return executeRestoreAudioAction(ctx, cfg, safeLocalPath, action, store)
}
tmpPath, err := downloadObjectToSiblingTemp(ctx, store, action.RemoteKey, safeLocalPath)
if err != nil {
return fmt.Errorf("download to temp file: %w", err)
@@ -120,6 +125,38 @@ func executeRestoreDownloadAction(
return nil
}
func executeRestoreAudioAction(
ctx context.Context,
cfg *config.Config,
safeLocalPath string,
action RestoreAction,
store storage.ObjectStore,
) error {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || cfg.Session == nil {
return fmt.Errorf("resolved s3 config and session are required")
}
spoolDir := artifacts.SessionSpoolRestoreAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID)
spoolPath := filepath.Join(spoolDir, filepath.Base(safeLocalPath))
cacheEnabled := cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio
_, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
Store: store,
Object: storage.ObjectInfo{
Key: action.RemoteKey,
Size: action.Size,
ETag: action.ETag,
},
Bucket: strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket),
CacheRoot: strings.TrimSpace(cfg.Pipeline.Cache.Root),
CacheEnabled: cacheEnabled,
SpoolPath: spoolPath,
DestPath: safeLocalPath,
})
if err != nil {
return fmt.Errorf("materialize audio: %w", err)
}
return nil
}
func downloadObjectToSiblingTemp(ctx context.Context, store storage.ObjectStore, remoteKey, destPath string) (string, error) {
if strings.TrimSpace(destPath) == "" {
return "", fmt.Errorf("destination path is required")

View File

@@ -18,10 +18,10 @@ import (
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
@@ -32,7 +32,7 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -64,17 +64,17 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -87,12 +87,77 @@ func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
}
}
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
func TestExecuteRestoreIncludeAudioUsesCacheAfterWorkspaceDeletion(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
audioKey := sessionPrefix + "audio/alice.flac"
seedRestoreObject(fake, audioKey, []byte("remote-audio"))
restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("first restore exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if got := fakeDownloadCount(fake, audioKey); got != 1 {
t.Fatalf("audio downloads after first restore = %d, want 1", got)
}
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
if err := os.RemoveAll(sessionRoot); err != nil {
t.Fatalf("remove session root: %v", err)
}
stdout.Reset()
stderr.Reset()
code = Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("second restore exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if got := fakeDownloadCount(fake, audioKey); got != 1 {
t.Fatalf("audio downloads after cached restore = %d, want still 1", got)
}
mustReadEquals(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "remote-audio")
}
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "manifest.json"), `{"session_id":"2026-04-26"}`)
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
report := mustReadRestoreReport(t, filepath.Join(sessionRoot, "reports", "restore-latest.json"))
if report.Execution.Downloaded != 3 {
t.Fatalf("report execution.downloaded = %d, want 3", report.Execution.Downloaded)
}
}
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
@@ -102,7 +167,7 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -121,10 +186,10 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
@@ -134,7 +199,7 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -145,12 +210,34 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
}
}
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustWriteTestFile(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# local previous recap\n")
restoreWithStoreAndRealPhases(t, fake)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# remote previous recap\n")
}
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
store := artifacts.NewLocalStore(workspaceRoot)
@@ -164,7 +251,7 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -180,10 +267,10 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
base := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, sessionPath)
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
toggled := &stagedManifestDownloadStore{
@@ -211,7 +298,7 @@ func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -295,9 +382,9 @@ func restoreWithStoreAndRealPhases(t *testing.T, objectStore storage.ObjectStore
executeRestorePlanFn = executeRestorePlan
}
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, sessionPath string) (*config.Config, string, string, string) {
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, campaignPath, sessionPath string) (*config.Config, string, string, string) {
t.Helper()
cfg, err := config.LoadWithSessionOptions(pipelinePath, sessionPath, config.SessionLoadOptions{})
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
@@ -325,6 +412,16 @@ func mustReadEquals(t *testing.T, path, want string) {
}
}
func fakeDownloadCount(fake *storage.FakeBackend, key string) int {
count := 0
for _, call := range fake.Downloads {
if call.Key == key {
count++
}
}
return count
}
type stagedManifestDownloadStore struct {
delegate *storage.FakeBackend
manifestKey string

View File

@@ -194,6 +194,9 @@ func restoreLocalRelativePathForKey(sessionPrefix, currentManifestKey, key strin
if cleanRel == config.PathArtifactsDirSegment || strings.HasPrefix(cleanRel, config.PathArtifactsDirSegment+"/") {
return cleanRel, true, nil
}
if cleanRel == config.PathPreviousDirSegment || strings.HasPrefix(cleanRel, config.PathPreviousDirSegment+"/") {
return cleanRel, true, nil
}
if includeAudio && (cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")) {
return cleanRel, true, nil
}
@@ -254,6 +257,35 @@ func classifyRestoreAction(
return action, nil
}
if restoreRelativePathIsAudio(localRelPath) {
if object.Size > 0 {
if info.Size() == object.Size {
action.Kind = RestoreActionSkipSame
action.SameLocal = true
action.Reason = "local audio size matches remote content"
return action, nil
}
if force {
action.Kind = RestoreActionDownload
action.Reason = "local audio differs (size mismatch); overwrite with --force"
return action, nil
}
action.Kind = RestoreActionConflict
action.Conflict = true
action.Reason = "local audio differs (size mismatch)"
return action, nil
}
if force {
action.Kind = RestoreActionDownload
action.Reason = "local audio exists; remote size unavailable; overwrite with --force"
return action, nil
}
action.Kind = RestoreActionConflict
action.Conflict = true
action.Reason = "local audio exists; remote size unavailable"
return action, nil
}
if object.Size > 0 && info.Size() != object.Size {
if force {
action.Kind = RestoreActionDownload
@@ -300,6 +332,15 @@ func classifyRestoreAction(
return action, nil
}
func restoreActionIsAudio(action RestoreAction) bool {
return restoreRelativePathIsAudio(action.LocalRelativePath)
}
func restoreRelativePathIsAudio(rel string) bool {
cleanRel := path.Clean(strings.TrimSpace(rel))
return cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")
}
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
if out == nil {
return fmt.Errorf("output writer is required")

View File

@@ -58,6 +58,54 @@ func TestRestorePlanIncludeAudio(t *testing.T) {
}
}
func TestRestorePlanExistingAudioUsesSizeWithoutRemoteChecksumDownload(t *testing.T) {
cfg := restorePlanConfig(t)
current := restorePlanCurrentState(t, cfg)
store := &storage.FakeBackend{}
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
seedRestoreObject(store, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
sessionRoot := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
mustWriteTestFile(t, filepath.Join(sessionRoot, "audio", "alice.flac"), "local")
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{IncludeAudio: true})
if err != nil {
t.Fatalf("buildRestorePlan() error = %v", err)
}
if len(store.Downloads) != 0 {
t.Fatalf("downloads = %d, want no remote checksum download for audio", len(store.Downloads))
}
actionByRel := map[string]RestoreAction{}
for _, action := range plan.Actions {
actionByRel[action.LocalRelativePath] = action
}
audioAction := actionByRel["audio/alice.flac"]
if audioAction.Kind != RestoreActionSkipSame {
t.Fatalf("audio action kind = %q, want %q", audioAction.Kind, RestoreActionSkipSame)
}
}
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
cfg := restorePlanConfig(t)
current := restorePlanCurrentState(t, cfg)
store := &storage.FakeBackend{}
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
seedRestoreObject(store, current.SessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
seedRestoreObject(store, current.SessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{})
if err != nil {
t.Fatalf("buildRestorePlan() error = %v", err)
}
got := actionRelPaths(plan.Actions)
want := []string{"manifest.json", "previous/artifacts/session_recap.md", "previous/manifest.json"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("action local paths = %#v, want %#v", got, want)
}
}
func TestRestorePlanClassifiesSameAndConflict(t *testing.T) {
cfg := restorePlanConfig(t)
current := restorePlanCurrentState(t, cfg)

View File

@@ -32,6 +32,9 @@ func TestExecuteRestoreHelp(t *testing.T) {
if !strings.Contains(out, "--include-audio") {
t.Fatalf("stdout = %q, want --include-audio flag", out)
}
if !strings.Contains(out, "--campaign") {
t.Fatalf("stdout = %q, want --campaign flag", out)
}
}
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
@@ -70,7 +73,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
}
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -78,6 +81,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
[]string{
"restore",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--dry-run",
@@ -116,11 +120,11 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "extra"}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "extra"}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -138,11 +142,11 @@ func TestExecuteRestoreFailsWhenStorageBackendNotConfigured(t *testing.T) {
})
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -166,11 +170,11 @@ func TestExecuteRestoreDiscoveryErrorSurfaced(t *testing.T) {
}
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -208,7 +212,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
restoreEnv(secretKeyEnv)
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
secretsDir := filepath.Join(t.TempDir(), "secrets")
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
@@ -261,6 +265,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
[]string{
"restore",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--dry-run",
@@ -307,10 +312,10 @@ func TestExecuteRestoreNonDryRunConflictFailsBeforeNYI(t *testing.T) {
}
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
@@ -359,10 +364,10 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
}
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
@@ -374,11 +379,12 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
}
}
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string) {
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string, string) {
t.Helper()
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := writeAppTestCampaignConfig(t, dir)
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace:
@@ -405,5 +411,5 @@ inputs:
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
return pipelinePath, sessionPath
return pipelinePath, campaignPath, sessionPath
}

View File

@@ -19,10 +19,10 @@ import (
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
fake := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
@@ -48,6 +48,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
[]string{
"restore",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", cfg.Session.SessionID,
},
@@ -86,6 +87,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
[]string{
"run-stage",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", cfg.Session.SessionID,
"--force",
@@ -155,6 +157,118 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
}
}
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
transcript:
source: narratio.transcript.trimmed
required: true
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: true
`)
fakeStore := &storage.FakeBackend{}
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, campaignPath, sessionPath)
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
seedRestoreObject(fakeStore, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
seedRestoreObject(fakeStore, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
restoreWithStoreAndRealPhases(t, fakeStore)
var stdout bytes.Buffer
var stderr bytes.Buffer
restoreCode := Execute(
[]string{
"restore",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", cfg.Session.SessionID,
},
&stdout,
&stderr,
)
if restoreCode != 0 {
t.Fatalf("restore exit code = %d, want 0; stderr=%q", restoreCode, stderr.String())
}
if stderr.Len() != 0 {
t.Fatalf("restore stderr = %q, want empty", stderr.String())
}
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustReadEquals(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), `{"segments":[]}`+"\n")
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "manifest.json"), `{"session_id":"2026-04-26"}`)
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
scriptoriumFake := &scriptorium.FakeRunner{}
origExecuteStagesFn := executeStagesFn
origObjectStoreFn := newObjectStoreFromConfigFn
objectStoreConstructed := false
t.Cleanup(func() {
executeStagesFn = origExecuteStagesFn
newObjectStoreFromConfigFn = origObjectStoreFn
})
executeStagesFn = func(ctx context.Context, cfg *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
if opts.Env == nil {
opts.Env = &Env{}
}
opts.Env.Scriptorium = scriptoriumFake
return executeStages(ctx, cfg, stages, opts)
}
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
objectStoreConstructed = true
return nil, context.Canceled
}
stdout.Reset()
stderr.Reset()
runStageCode := Execute(
[]string{
"run-stage",
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", cfg.Session.SessionID,
"--force",
"--artifacts", "session_recap",
"analyze",
},
&stdout,
&stderr,
)
if runStageCode != 0 {
t.Fatalf("run-stage exit code = %d, want 0; stderr=%q", runStageCode, stderr.String())
}
if stderr.Len() != 0 {
t.Fatalf("run-stage stderr = %q, want empty", stderr.String())
}
if objectStoreConstructed {
t.Fatal("analyze run-stage should not construct object store for previous-session input resolution")
}
if len(scriptoriumFake.RunRequests) != 1 {
t.Fatalf("scriptorium run requests = %d, want 1", len(scriptoriumFake.RunRequests))
}
req := scriptoriumFake.RunRequests[0]
if got := req.InputPaths["transcript"]; got != filepath.Join(sessionRoot, "transcripts", "trimmed.json") {
t.Fatalf("transcript input = %q, want trimmed transcript path", got)
}
if got := req.InputPaths["previous_recap"]; got != filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md") {
t.Fatalf("previous_recap input = %q, want restored previous cache path", got)
}
}
func restoreWorkflowManifestJSON(t *testing.T, sessionID, campaign string) []byte {
t.Helper()
store := &manifest.LocalStore{}
@@ -176,3 +290,15 @@ func restoreWorkflowManifestJSON(t *testing.T, sessionID, campaign string) []byt
}
return data
}
func appendRestoreWorkflowScriptoriumConfig(t *testing.T, pipelinePath, extra string) {
t.Helper()
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
if err != nil {
t.Fatalf("open pipeline config for append: %v", err)
}
defer f.Close()
if _, err := f.WriteString(extra); err != nil {
t.Fatalf("append pipeline config: %v", err)
}
}

View File

@@ -17,13 +17,17 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var force bool
var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution")
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
@@ -33,17 +37,9 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("resume: unexpected positional arguments")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("resume: %w", err)

View File

@@ -15,7 +15,7 @@ import (
func TestResumeStartsAfterCompletedStages(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -32,7 +32,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil {
t.Fatalf("Resume() error = %v", err)
}
@@ -51,7 +51,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
func TestResumeNoRemainingStages(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -64,7 +64,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
}
var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil {
t.Fatalf("Resume() error = %v", err)
}
@@ -80,7 +80,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
@@ -93,7 +93,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
}
var out bytes.Buffer
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force"}, &out)
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &out)
if err != nil {
t.Fatalf("Resume() error = %v", err)
}
@@ -104,14 +104,14 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
if err != nil {
t.Fatalf("RunStage() error = %v", err)
}
@@ -134,7 +134,7 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
func TestRunStageSkipAndForce(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
@@ -148,7 +148,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
}
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
if err != nil {
t.Fatalf("RunStage() error = %v", err)
}
@@ -157,7 +157,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
}
out.Reset()
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
if err != nil {
t.Fatalf("RunStage(force) error = %v", err)
}
@@ -168,7 +168,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
@@ -184,7 +184,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
}
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
if err != nil {
t.Fatalf("RunStage(force) error = %v", err)
}
@@ -203,7 +203,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
}
out.Reset()
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
err = Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
if err != nil {
t.Fatalf("Resume() error = %v", err)
}
@@ -214,13 +214,13 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
func TestRunStageTrimExecutes(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "trim"}, &out)
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "trim"}, &out)
if err != nil {
t.Fatalf("RunStage(trim) error = %v", err)
}
@@ -243,13 +243,13 @@ func TestRunStageTrimExecutes(t *testing.T) {
func TestRunStageNormalizeExecutes(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "normalize"}, &out)
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &out)
if err != nil {
t.Fatalf("RunStage(normalize) error = %v", err)
}

View File

@@ -15,13 +15,17 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var force bool
var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
@@ -31,17 +35,9 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("run: unexpected positional arguments")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
if err != nil {
return fmt.Errorf("run: %w", err)

View File

@@ -15,13 +15,17 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var force bool
var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
@@ -31,53 +35,34 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 1 {
return fmt.Errorf("run-stage: expected exactly one stage name")
}
stageName := fs.Arg(0)
normalizedArtifacts, err := selectedArtifacts.Normalize()
if err != nil {
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
}
stageName := fs.Arg(0)
if len(normalizedArtifacts) > 0 && stageName != "analyze" {
return fmt.Errorf("run-stage: --artifacts is only supported for stage \"analyze\"")
}
stages, err := BuildSingleStagePlan(stageName)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("run-stage: %w", err)
}
if err := validateSelectedAnalyzeArtifacts(cfg, normalizedArtifacts); err != nil {
return fmt.Errorf("run-stage: %w", err)
}
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
summary, err := runSingleStageCommand(ctx, singleStageCommand{
CommandName: "run-stage",
StageName: stageName,
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
SessionID: sessionID,
PreviousSessionID: previousSessionID,
Force: force,
SelectedArtifacts: normalizedArtifacts,
})
if err != nil {
return fmt.Errorf("run-stage: %w", err)
return err
}
_, err = fmt.Fprintf(
out,
"narratio run-stage: stage=%s executed=%d skipped=%d force=%t; manifest=%s\n",
stages[0].Name(),
stageName,
len(summary.Executed),
len(summary.Skipped),
force,
@@ -85,3 +70,99 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
)
return err
}
// Analyze force-runs the analyze stage.
func Analyze(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("analyze", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var pipelinePath string
var campaignPath string
var sessionPath string
var sessionID string
var previousSessionID string
var selectedArtifacts artifactSelectionFlag
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("analyze: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("analyze: unexpected positional arguments")
}
normalizedArtifacts, err := selectedArtifacts.Normalize()
if err != nil {
return fmt.Errorf("analyze: invalid --artifacts: %w", err)
}
summary, err := runSingleStageCommand(ctx, singleStageCommand{
CommandName: "analyze",
StageName: "analyze",
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
SessionID: sessionID,
PreviousSessionID: previousSessionID,
Force: true,
SelectedArtifacts: normalizedArtifacts,
})
if err != nil {
return err
}
_, err = fmt.Fprintf(
out,
"narratio analyze: executed=%d skipped=%d force=true; manifest=%s\n",
len(summary.Executed),
len(summary.Skipped),
summary.ManifestPath,
)
return err
}
type singleStageCommand struct {
CommandName string
StageName string
PipelinePath string
CampaignPath string
SessionPath string
SessionID string
PreviousSessionID string
Force bool
SelectedArtifacts []string
}
func runSingleStageCommand(ctx context.Context, req singleStageCommand) (*RunSummary, error) {
stages, err := BuildSingleStagePlan(req.StageName)
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
cfg, err := loadCommandConfig(ctx, req.PipelinePath, req.CampaignPath, req.SessionPath, config.SessionLoadOptions{
SessionID: req.SessionID,
PreviousSessionID: req.PreviousSessionID,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
if err := config.Validate(cfg); err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
if err := validateSelectedAnalyzeArtifacts(cfg, req.SelectedArtifacts); err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
summary, err := executeStagesFn(ctx, cfg, stages, RunOptions{
Force: req.Force,
SelectedArtifacts: req.SelectedArtifacts,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
}
return summary, nil
}

View File

@@ -87,12 +87,19 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
env.Storage = &storage.NoopBackend{}
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
objectStore, err := newCommandObjectStore(ctx, env.Config, nil)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
return nil, err
}
env.ObjectStore = objectStore
}
if needsRemoteLocksForRun(env.Config, stages) {
locks, err := loadEffectiveLocks(ctx, env.Config, env.ObjectStore)
if err != nil {
return nil, fmt.Errorf("load remote archive locks: %w", err)
}
applyEffectiveLocks(env.Config, locks.All)
}
if env.Notifier == nil {
env.Notifier = &notify.NoopSender{}
}
@@ -555,6 +562,12 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
return true
}
if stageRequested("prepare") {
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
if len(requirements) > 0 && strings.TrimSpace(cfg.Session.PreviousSessionID) != "" {
return true
}
}
if !stageRequested("archive") {
return false
}
@@ -569,3 +582,36 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
}
return true
}
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return false
}
archiveRequested := false
for _, s := range stages {
if s != nil && s.Name() == "archive" {
archiveRequested = true
break
}
}
if !archiveRequested {
return false
}
if cfg.Pipeline.Archive == nil {
return false
}
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
return false
}
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
return false
}
return cfg.Pipeline.Storage.S3 != nil
}
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
return nil
}
return cfg.Pipeline.Scriptorium.Artifacts
}

View File

@@ -199,6 +199,55 @@ func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T)
}
}
func TestNeedsObjectStoreForRunPrepareWithPreviousRequirements(t *testing.T) {
tests := []struct {
name string
previousSessionID string
want bool
}{
{
name: "previous session configured",
previousSessionID: "2026-05-10",
want: true,
},
{
name: "previous session missing",
previousSessionID: "",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Scriptorium: &config.ScriptoriumConfig{
Artifacts: map[string]config.ScriptoriumArtifactConfig{
"session_recap": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"previous_recap": {
Source: "narratio.previous_session.artifact.session_recap",
Required: true,
},
},
},
},
},
},
Session: &config.SessionConfig{
PreviousSessionID: tt.previousSessionID,
},
}
got := needsObjectStoreForRun(cfg, []stage.Stage{countingStage{name: "prepare", runs: new(int)}})
if got != tt.want {
t.Fatalf("needsObjectStoreForRun() = %v, want %v", got, tt.want)
}
})
}
}
func TestExecuteStagesArchiveFailsWhenRequiredRecapPromotionMissingForSelectedArtifacts(t *testing.T) {
cfg := testConfig(t)
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
@@ -945,10 +994,12 @@ func testConfig(t *testing.T) *config.Config {
workspace := t.TempDir()
cfgDir := t.TempDir()
sessionPath := filepath.Join(cfgDir, "session.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
mustWriteFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n")
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -956,8 +1007,27 @@ func testConfig(t *testing.T) *config.Config {
return &config.Config{
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
@@ -974,6 +1044,7 @@ func testConfig(t *testing.T) *config.Config {
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace:
root: ` + t.TempDir() + `
@@ -983,6 +1054,12 @@ analyzer:
timeout: 20m
notification:
timeout: 10s
`
campaignYAML := `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
@@ -993,6 +1070,7 @@ inputs:
glossary_file: ./glossary.yml
`
mustWriteFile(t, pipelinePath, pipelineYAML)
mustWriteFile(t, campaignPath, campaignYAML)
mustWriteFile(t, sessionPath, sessionYAML)
cfg, err := config.Load(pipelinePath, sessionPath)

View File

@@ -4,16 +4,18 @@ import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestPlanUsesDiscoveredSessionTemplateWithSessionID(t *testing.T) {
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionTemplate := `session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
@@ -25,18 +27,17 @@ inputs:
t.Fatalf("write session template: %v", err)
}
cwd := filepath.Dir(sessionPath)
originalWD, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(cwd); err != nil {
t.Fatalf("Chdir(%q): %v", cwd, err)
}
t.Cleanup(func() { _ = os.Chdir(originalWD) })
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
config.DefaultSessionConfigSearchPaths = []string{sessionPath}
t.Cleanup(func() { config.DefaultSessionConfigSearchPaths = origSessionDefaults })
var out bytes.Buffer
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session-id", "2026-04-04"}, &out); err != nil {
if err := Plan(context.Background(), []string{
"--config", pipelinePath,
"--campaign", campaignPath,
"--session-id", "2026-04-04",
"--previous-session-id", "2026-03-28",
}, &out); err != nil {
t.Fatalf("Plan() error = %v", err)
}
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
@@ -46,10 +47,10 @@ inputs:
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}
@@ -58,12 +59,45 @@ func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
}
}
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionYAML := `session_id: 2026-05-03
previous_session_id: 2026-04-26
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
err := Plan(context.Background(), []string{
"--config", pipelinePath,
"--campaign", campaignPath,
"--session", sessionPath,
"--session-id", "2026-05-03",
"--previous-session-id", "2026-04-25",
}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "previous_session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
if err != nil {
t.Fatalf("RunStage() error = %v", err)
}
@@ -73,7 +107,7 @@ func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
}
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
_, err := resolveSessionConfigPathWithCandidates("", []string{"/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}

View File

@@ -19,6 +19,22 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
return explicit, nil
}
resolved, err := discoverSessionConfigPathWithCandidates(candidates)
if err != nil {
return "", err
}
if resolved.Path != "" {
return resolved.Path, nil
}
return "", missingSessionConfigError(resolved.Searched, "")
}
type sessionConfigDiscovery struct {
Path string
Searched []string
}
func discoverSessionConfigPathWithCandidates(candidates []string) (sessionConfigDiscovery, error) {
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
@@ -31,19 +47,32 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
return sessionConfigDiscovery{Path: filepath.Clean(path), Searched: ordered}, nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default session config %q: %w", path, err)
return sessionConfigDiscovery{}, fmt.Errorf("check default session config %q: %w", path, err)
}
return sessionConfigDiscovery{Searched: ordered}, nil
}
func missingSessionConfigError(searched []string, remoteDetail string) error {
ordered := append([]string(nil), searched...)
if len(ordered) == 0 {
return "", fmt.Errorf("no session config path provided and no default locations configured")
if strings.TrimSpace(remoteDetail) != "" {
return fmt.Errorf("no session config path provided and no default locations configured; %s", remoteDetail)
}
return fmt.Errorf("no session config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
msg := fmt.Sprintf(
"no session config path provided and no default session config found; searched: %s",
strings.Join(ordered, ", "),
)
if strings.TrimSpace(remoteDetail) != "" {
msg += "; " + strings.TrimSpace(remoteDetail)
}
msg += "; pass --session to use an explicit path"
return fmt.Errorf("%s", msg)
}

View File

@@ -1,67 +0,0 @@
package app
import (
"context"
"flag"
"fmt"
"io"
"sort"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
// Status reads and prints stage statuses from an existing manifest.
func Status(ctx context.Context, args []string, out io.Writer) error {
fs := flag.NewFlagSet("status", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var manifestPath string
fs.StringVar(&manifestPath, "manifest", "", "path to manifest.json")
if err := fs.Parse(args); err != nil {
return fmt.Errorf("status: invalid flags: %w", err)
}
if fs.NArg() != 0 {
return fmt.Errorf("status: unexpected positional arguments")
}
if manifestPath == "" {
return fmt.Errorf("status: --manifest is required")
}
store := &manifest.LocalStore{}
m, err := store.Load(ctx, manifestPath)
if err != nil {
return fmt.Errorf("status: %w", err)
}
if _, err := fmt.Fprintf(out, "session_id: %s\n", m.SessionID); err != nil {
return err
}
if _, err := fmt.Fprintf(out, "updated_at: %s\n", m.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil {
return err
}
if len(m.Stages) == 0 {
_, err := fmt.Fprintln(out, "stages: no stages recorded")
return err
}
if _, err := fmt.Fprintln(out, "stages:"); err != nil {
return err
}
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
status := m.Stages[name].Status
if _, err := fmt.Fprintf(out, "- %s: %s\n", name, status); err != nil {
return err
}
}
return nil
}

View File

@@ -18,11 +18,15 @@ const (
ArtifactTranscriptFull = "narratio.transcript.full"
ArtifactTranscriptTrimmed = "narratio.transcript.trimmed"
ArtifactBoundsSession = "narratio.bounds.session"
ArtifactProvenancePreviousCacheManifestInput = "manifest.inputs.previous_cache"
ArtifactProvenancePreviousCacheFilesystem = "current_session.previous_cache"
)
// ErrSessionArtifactNotFound is returned when no readable artifact exists for a known ID.
var ErrSessionArtifactNotFound = errors.New("session artifact not found")
var configuredArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.[a-z][a-z0-9_]*$`)
var previousSessionArtifactSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
type artifactContentKind string
@@ -118,6 +122,21 @@ func IsConfiguredArtifactSource(source string) bool {
return configuredArtifactSourceRE.MatchString(strings.TrimSpace(source))
}
// IsPreviousSessionArtifactSource returns true when source is narratio.previous_session.artifact.<name>.
func IsPreviousSessionArtifactSource(source string) bool {
_, ok := PreviousSessionArtifactName(source)
return ok
}
// PreviousSessionArtifactName extracts <name> from narratio.previous_session.artifact.<name>.
func PreviousSessionArtifactName(source string) (string, bool) {
matches := previousSessionArtifactSourceRE.FindStringSubmatch(strings.TrimSpace(source))
if len(matches) != 2 {
return "", false
}
return matches[1], true
}
// ResolveSessionArtifact resolves a symbolic source to a readable local session artifact path.
// Resolution order is manifest producer outputs first, then canonical session path fallback.
func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source string) (ResolvedSessionArtifact, error) {
@@ -170,6 +189,9 @@ func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source str
// configured narratio.artifact.<name> sources through runtime catalog availability.
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
normalized := strings.TrimSpace(source)
if IsPreviousSessionArtifactSource(normalized) {
return ResolvePreviousSessionArtifactWithCatalog(paths, m, normalized, catalog)
}
if !IsConfiguredArtifactSource(normalized) {
return ResolveSessionArtifact(paths, m, normalized)
}
@@ -195,6 +217,72 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
}, nil
}
// ResolvePreviousSessionArtifactWithCatalog resolves one canonical previous-session source id
// to the prepared current-session previous-cache path.
func ResolvePreviousSessionArtifactWithCatalog(
paths SessionPaths,
m *manifest.Manifest,
source string,
catalog *ArtifactCatalog,
) (ResolvedSessionArtifact, error) {
artifactName, ok := PreviousSessionArtifactName(source)
if !ok {
return ResolvedSessionArtifact{}, fmt.Errorf("unsupported previous-session artifact source %q", source)
}
if catalog == nil {
return ResolvedSessionArtifact{}, fmt.Errorf("previous-session artifact source %q requires runtime artifact catalog", source)
}
configuredSourceID := ConfiguredArtifactSourceID(artifactName)
entry, ok := catalog.Lookup(configuredSourceID)
if !ok {
return ResolvedSessionArtifact{}, fmt.Errorf("unsupported previous-session artifact source %q", source)
}
candidates := previousSessionCacheCandidatePaths(paths, entry.CanonicalRelPath)
if len(candidates) == 0 {
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: source}
}
manifestInputPaths := manifestInputPathSet(paths, m)
fallback := ""
for _, candidate := range candidates {
exists, isDir, statErr := pathExists(candidate)
if statErr != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", candidate, statErr)
}
if !exists || isDir {
continue
}
if err := validateResolvedContent(candidate, contentText); err != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", source, err)
}
if _, ok := manifestInputPaths[candidate]; ok {
return ResolvedSessionArtifact{
ID: source,
Path: candidate,
ProducerStage: "prepare",
OutputKind: "previous_session_artifact",
Provenance: ArtifactProvenancePreviousCacheManifestInput,
}, nil
}
if fallback == "" {
fallback = candidate
}
}
if fallback != "" {
return ResolvedSessionArtifact{
ID: source,
Path: fallback,
ProducerStage: "prepare",
OutputKind: "previous_session_artifact",
Provenance: ArtifactProvenancePreviousCacheFilesystem,
}, nil
}
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: source}
}
func manifestArtifactCandidates(paths SessionPaths, m *manifest.Manifest, spec artifactSpec) []ResolvedSessionArtifact {
if m == nil || len(m.Stages) == 0 || spec.ProducerStage == "" || spec.OutputKind == "" {
return nil
@@ -243,6 +331,50 @@ func dedupeResolvedArtifacts(values []ResolvedSessionArtifact) []ResolvedSession
return out
}
func previousSessionCacheCandidatePaths(paths SessionPaths, canonicalRelPath string) []string {
trimmed := strings.TrimSpace(canonicalRelPath)
if trimmed == "" {
return nil
}
normalized := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
if normalized == "." || normalized == "" || normalized == ".." || strings.HasPrefix(normalized, "../") || strings.HasPrefix(normalized, "/") {
return nil
}
relCandidates := []string{normalized}
const artifactsPrefix = "artifacts/"
if strings.HasPrefix(normalized, artifactsPrefix) && len(normalized) > len(artifactsPrefix) {
relCandidates = append(relCandidates, strings.TrimPrefix(normalized, artifactsPrefix))
}
out := make([]string, 0, len(relCandidates))
seen := map[string]struct{}{}
for _, rel := range relCandidates {
abs := filepath.Clean(SessionPreviousArtifactPath(paths, rel))
if _, ok := seen[abs]; ok {
continue
}
seen[abs] = struct{}{}
out = append(out, abs)
}
return out
}
func manifestInputPathSet(paths SessionPaths, m *manifest.Manifest) map[string]struct{} {
if m == nil || len(m.Inputs) == 0 {
return nil
}
out := make(map[string]struct{}, len(m.Inputs))
for _, in := range m.Inputs {
resolved := filepath.Clean(ResolveSessionLocalPathForRead(paths, in.Path))
if strings.TrimSpace(resolved) == "" {
continue
}
out[resolved] = struct{}{}
}
return out
}
func pathExists(path string) (exists bool, isDir bool, err error) {
info, err := os.Stat(path)
if err == nil {

View File

@@ -4,6 +4,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
@@ -45,6 +46,58 @@ func TestNormalizeSessionArtifactSource(t *testing.T) {
}
}
func TestPreviousSessionArtifactSourceHelpers(t *testing.T) {
tests := []struct {
name string
source string
wantName string
wantMatch bool
}{
{
name: "valid",
source: "narratio.previous_session.artifact.session_recap",
wantName: "session_recap",
wantMatch: true,
},
{
name: "valid with surrounding whitespace",
source: " narratio.previous_session.artifact.quest_log ",
wantName: "quest_log",
wantMatch: true,
},
{
name: "missing name",
source: "narratio.previous_session.artifact.",
wantMatch: false,
},
{
name: "invalid key characters",
source: "narratio.previous_session.artifact.session-recap",
wantMatch: false,
},
{
name: "wrong prefix",
source: "narratio.previous.artifact.session_recap",
wantMatch: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
name, ok := PreviousSessionArtifactName(tt.source)
if ok != tt.wantMatch {
t.Fatalf("PreviousSessionArtifactName(%q) ok = %t, want %t", tt.source, ok, tt.wantMatch)
}
if name != tt.wantName {
t.Fatalf("PreviousSessionArtifactName(%q) name = %q, want %q", tt.source, name, tt.wantName)
}
if got := IsPreviousSessionArtifactSource(tt.source); got != tt.wantMatch {
t.Fatalf("IsPreviousSessionArtifactSource(%q) = %t, want %t", tt.source, got, tt.wantMatch)
}
})
}
}
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
@@ -266,3 +319,131 @@ func TestResolveSessionArtifactWithCatalogUnsupportedConfiguredSourceFails(t *te
t.Fatalf("error = %q, want unsupported artifact source", err.Error())
}
}
func TestResolvePreviousSessionArtifactWithCatalogPrefersManifestInputRecord(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
manifestBackedPath := SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
fallbackPath := SessionPreviousArtifactPath(paths, "session_recap.md")
if err := os.MkdirAll(filepath.Dir(manifestBackedPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.MkdirAll(filepath.Dir(fallbackPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(manifestBackedPath, []byte("recap from manifest input\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if err := os.WriteFile(fallbackPath, []byte("recap fallback\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
catalog := NewArtifactCatalog()
if err := catalog.RegisterConfiguredArtifacts(
map[string]ConfiguredArtifactDefinition{
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
},
nil,
); err != nil {
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
}
m := manifest.New("session", time.Now().UTC())
m.Inputs = []manifest.InputRecord{
{Kind: "previous_artifact", Path: manifestBackedPath},
}
resolved, err := ResolvePreviousSessionArtifactWithCatalog(
paths,
m,
"narratio.previous_session.artifact.session_recap",
catalog,
)
if err != nil {
t.Fatalf("ResolvePreviousSessionArtifactWithCatalog() error = %v", err)
}
if resolved.Path != manifestBackedPath {
t.Fatalf("resolved path = %q, want %q", resolved.Path, manifestBackedPath)
}
if resolved.Provenance != ArtifactProvenancePreviousCacheManifestInput {
t.Fatalf("provenance = %q, want %q", resolved.Provenance, ArtifactProvenancePreviousCacheManifestInput)
}
}
func TestResolvePreviousSessionArtifactWithCatalogFallsBackToPreparedCachePath(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
fallbackPath := SessionPreviousArtifactPath(paths, "session_recap.md")
if err := os.MkdirAll(filepath.Dir(fallbackPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(fallbackPath, []byte("recap fallback\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
catalog := NewArtifactCatalog()
if err := catalog.RegisterConfiguredArtifacts(
map[string]ConfiguredArtifactDefinition{
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
},
nil,
); err != nil {
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
}
resolved, err := ResolvePreviousSessionArtifactWithCatalog(
paths,
nil,
"narratio.previous_session.artifact.session_recap",
catalog,
)
if err != nil {
t.Fatalf("ResolvePreviousSessionArtifactWithCatalog() error = %v", err)
}
if resolved.Path != fallbackPath {
t.Fatalf("resolved path = %q, want %q", resolved.Path, fallbackPath)
}
if resolved.Provenance != ArtifactProvenancePreviousCacheFilesystem {
t.Fatalf("provenance = %q, want %q", resolved.Provenance, ArtifactProvenancePreviousCacheFilesystem)
}
}
func TestPreviousSessionCacheCandidatePathsStripsArtifactsPrefix(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
got := previousSessionCacheCandidatePaths(paths, "artifacts/session_recap.md")
want := []string{filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")}
if !reflect.DeepEqual(got, want) {
t.Fatalf("previousSessionCacheCandidatePaths() = %#v, want %#v", got, want)
}
}
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
catalog := NewArtifactCatalog()
if err := catalog.RegisterConfiguredArtifacts(
map[string]ConfiguredArtifactDefinition{
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
},
nil,
); err != nil {
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
}
_, err := ResolvePreviousSessionArtifactWithCatalog(
paths,
nil,
"narratio.previous_session.artifact.session_recap",
catalog,
)
if err == nil {
t.Fatal("expected error, got nil")
}
if !errors.Is(err, ErrSessionArtifactNotFound) {
t.Fatalf("errors.Is(err, ErrSessionArtifactNotFound)=false; err=%v", err)
}
}

View File

@@ -73,6 +73,8 @@ func (s *LocalStore) ensureLayout(paths SessionPaths) (SessionPaths, error) {
paths.LogsDir,
paths.CurrentDir,
paths.RunsDir,
paths.PreviousDir,
paths.PreviousArtifactsDir,
}
for _, dir := range dirs {

View File

@@ -27,10 +27,15 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
checkDirExists(t, paths.LogsDir)
checkDirExists(t, paths.CurrentDir)
checkDirExists(t, paths.RunsDir)
checkDirExists(t, paths.PreviousDir)
checkDirExists(t, paths.PreviousArtifactsDir)
if filepath.Base(paths.ManifestPath) != "manifest.json" {
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
}
if filepath.Base(paths.PreviousManifestPath) != "manifest.json" {
t.Fatalf("PreviousManifestPath = %q, want basename manifest.json", paths.PreviousManifestPath)
}
if filepath.Base(paths.LockPath) != ".lock" {
t.Fatalf("LockPath = %q, want basename .lock", paths.LockPath)
}

View File

@@ -1,7 +1,10 @@
package artifacts
import (
"fmt"
"path"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
@@ -23,6 +26,9 @@ type SessionPaths struct {
LogsDir string
CurrentDir string
RunsDir string
PreviousDir string
PreviousManifestPath string
PreviousArtifactsDir string
ManifestPath string
LockPath string
}
@@ -42,6 +48,29 @@ func SessionRunsDirForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathRunsDirSegment)
}
// SessionPreviousDirForCampaign returns the canonical previous-session state directory.
func SessionPreviousDirForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathPreviousDirSegment)
}
// SessionPreviousManifestPathForCampaign returns the canonical previous-session manifest cache path.
func SessionPreviousManifestPathForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionPreviousDirForCampaign(rootDir, campaign, sessionID), config.PathManifestFile)
}
// SessionPreviousArtifactsDirForCampaign returns the canonical previous-session artifact cache directory.
func SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionPreviousDirForCampaign(rootDir, campaign, sessionID), config.PathArtifactsDirSegment)
}
// SessionPreviousArtifactPathForCampaign returns a path under previous/artifacts for one artifact.
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
return filepath.Join(
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)),
)
}
// SessionRunRootForCampaign returns the canonical run root under runs/{run_id}.
func SessionRunRootForCampaign(rootDir, campaign, sessionID, runID string) string {
return filepath.Join(SessionRunsDirForCampaign(rootDir, campaign, sessionID), runID)
@@ -62,6 +91,92 @@ func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
}
// SessionSpoolDir returns the campaign/session scoped local spool root.
func SessionSpoolDir(spoolRoot, campaign, sessionID string) string {
return filepath.Join(spoolRoot, campaign, sessionID)
}
// SessionSpoolRestoreAudioDir returns the local spool audio path for restore downloads.
func SessionSpoolRestoreAudioDir(spoolRoot, campaign, sessionID string) string {
return filepath.Join(SessionSpoolDir(spoolRoot, campaign, sessionID), "restore", config.PathAudioDirSegment)
}
// S3AudioCachePath returns the durable local cache path for one S3 audio object.
func S3AudioCachePath(cacheRoot, bucket, key string) (string, error) {
root := filepath.Clean(strings.TrimSpace(cacheRoot))
if root == "." || root == "" {
return "", fmt.Errorf("cache root is required")
}
bucket = strings.Trim(strings.TrimSpace(bucket), "/")
if bucket == "" || bucket == "." || bucket == ".." || strings.Contains(bucket, "/") || strings.Contains(bucket, `\`) {
return "", fmt.Errorf("bucket is required and must be a single path segment")
}
rawKey := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
if strings.HasPrefix(rawKey, "/") {
return "", fmt.Errorf("s3 key must not be absolute")
}
cleanKey := cleanCacheS3Key(rawKey)
if cleanKey == "" {
return "", fmt.Errorf("s3 key is required")
}
if cleanKey == ".." || strings.HasPrefix(cleanKey, "../") || strings.HasPrefix(cleanKey, "/") {
return "", fmt.Errorf("s3 key must not escape cache root")
}
return filepath.Join(root, "s3", bucket, filepath.FromSlash(cleanKey)), nil
}
// S3AudioCacheNamespaceDir returns the durable local cache namespace for all
// Narratio S3 audio objects under one bucket/root prefix.
func S3AudioCacheNamespaceDir(cacheRoot, bucket, rootPrefix string) (string, error) {
sentinelKey := path.Join(cleanS3PathPart(rootPrefix), config.S3CampaignsSegment, ".narratio-cache-sentinel")
sentinelPath, err := S3AudioCachePath(cacheRoot, bucket, sentinelKey)
if err != nil {
return "", err
}
return filepath.Dir(sentinelPath), nil
}
func cleanCacheS3Key(key string) string {
normalized := strings.ReplaceAll(strings.TrimSpace(key), `\`, "/")
normalized = strings.Trim(normalized, "/")
if normalized == "" {
return ""
}
return path.Clean(normalized)
}
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
func SessionPreviousDir(paths SessionPaths) string {
return paths.PreviousDir
}
// SessionPreviousManifestPath returns the previous-session manifest path for already-resolved session paths.
func SessionPreviousManifestPath(paths SessionPaths) string {
return paths.PreviousManifestPath
}
// SessionPreviousArtifactsDir returns the previous-session artifact directory for already-resolved session paths.
func SessionPreviousArtifactsDir(paths SessionPaths) string {
return paths.PreviousArtifactsDir
}
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(previousArtifactCacheRelativePath(artifactRelativePath)))
}
func previousArtifactCacheRelativePath(artifactRelativePath string) string {
rel := filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimSpace(artifactRelativePath))))
if rel == "." {
return ""
}
const artifactsPrefix = "artifacts/"
if strings.HasPrefix(rel, artifactsPrefix) && len(rel) > len(artifactsPrefix) {
return strings.TrimPrefix(rel, artifactsPrefix)
}
return rel
}
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
root := SessionWorkDirForCampaign(workspaceRoot, campaign, sessionID)
return buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root)
@@ -84,6 +199,9 @@ func buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root string)
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
CurrentDir: filepath.Join(root, config.PathCurrentDirSegment),
RunsDir: filepath.Join(root, config.PathRunsDirSegment),
PreviousDir: filepath.Join(root, config.PathPreviousDirSegment),
PreviousManifestPath: filepath.Join(root, config.PathPreviousDirSegment, config.PathManifestFile),
PreviousArtifactsDir: filepath.Join(root, config.PathPreviousDirSegment, config.PathArtifactsDirSegment),
ManifestPath: filepath.Join(root, config.PathManifestFile),
LockPath: filepath.Join(root, config.PathLockFile),
}

View File

@@ -49,6 +49,63 @@ func TestSessionRunManifestPathForCampaign(t *testing.T) {
}
}
func TestSessionPreviousPathsForCampaign(t *testing.T) {
root := "/tmp/workspace"
previousDir := SessionPreviousDirForCampaign(root, "forsaken", "2026-04-19")
wantPreviousDir := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous")
if previousDir != wantPreviousDir {
t.Fatalf("SessionPreviousDirForCampaign() = %q, want %q", previousDir, wantPreviousDir)
}
manifestPath := SessionPreviousManifestPathForCampaign(root, "forsaken", "2026-04-19")
wantManifestPath := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "manifest.json")
if manifestPath != wantManifestPath {
t.Fatalf("SessionPreviousManifestPathForCampaign() = %q, want %q", manifestPath, wantManifestPath)
}
artifactsDir := SessionPreviousArtifactsDirForCampaign(root, "forsaken", "2026-04-19")
wantArtifactsDir := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "artifacts")
if artifactsDir != wantArtifactsDir {
t.Fatalf("SessionPreviousArtifactsDirForCampaign() = %q, want %q", artifactsDir, wantArtifactsDir)
}
artifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "session_recap.md")
wantArtifactPath := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "artifacts", "session_recap.md")
if artifactPath != wantArtifactPath {
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
}
archiveRelativeArtifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "artifacts/session_recap.md")
if archiveRelativeArtifactPath != wantArtifactPath {
t.Fatalf("SessionPreviousArtifactPathForCampaign(archive-relative) = %q, want %q", archiveRelativeArtifactPath, wantArtifactPath)
}
}
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
paths := buildSessionPaths("/tmp/workspace", "forsaken", "2026-04-19")
if got := SessionPreviousDir(paths); got != paths.PreviousDir {
t.Fatalf("SessionPreviousDir() = %q, want %q", got, paths.PreviousDir)
}
if got := SessionPreviousManifestPath(paths); got != paths.PreviousManifestPath {
t.Fatalf("SessionPreviousManifestPath() = %q, want %q", got, paths.PreviousManifestPath)
}
if got := SessionPreviousArtifactsDir(paths); got != paths.PreviousArtifactsDir {
t.Fatalf("SessionPreviousArtifactsDir() = %q, want %q", got, paths.PreviousArtifactsDir)
}
got := SessionPreviousArtifactPath(paths, "quest_log.json")
want := filepath.Join(paths.PreviousArtifactsDir, "quest_log.json")
if got != want {
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
}
got = SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
want = filepath.Join(paths.PreviousArtifactsDir, "session_recap.md")
if got != want {
t.Fatalf("SessionPreviousArtifactPath(archive-relative) = %q, want %q", got, want)
}
}
func TestSessionSpoolAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
@@ -57,3 +114,66 @@ func TestSessionSpoolAudioDir(t *testing.T) {
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
}
}
func TestSessionSpoolDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolDir(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "forsaken", "2026-04-19")
if got != want {
t.Fatalf("SessionSpoolDir() = %q, want %q", got, want)
}
}
func TestSessionSpoolRestoreAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolRestoreAudioDir(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "forsaken", "2026-04-19", "restore", "audio")
if got != want {
t.Fatalf("SessionSpoolRestoreAudioDir() = %q, want %q", got, want)
}
}
func TestS3AudioCachePath(t *testing.T) {
got, err := S3AudioCachePath("/var/cache/narratio", "my-dnd-archive", "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac")
if err != nil {
t.Fatalf("S3AudioCachePath() error = %v", err)
}
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns", "forsaken", "sessions", "2026-04-19", "audio", "alice.flac")
if got != want {
t.Fatalf("S3AudioCachePath() = %q, want %q", got, want)
}
}
func TestS3AudioCacheNamespaceDir(t *testing.T) {
got, err := S3AudioCacheNamespaceDir("/var/cache/narratio", "my-dnd-archive", "dnd")
if err != nil {
t.Fatalf("S3AudioCacheNamespaceDir() error = %v", err)
}
want := filepath.Join("/var/cache/narratio", "s3", "my-dnd-archive", "dnd", "campaigns")
if got != want {
t.Fatalf("S3AudioCacheNamespaceDir() = %q, want %q", got, want)
}
}
func TestS3AudioCachePathRejectsUnsafeInputs(t *testing.T) {
tests := []struct {
name string
root string
bucket string
key string
}{
{name: "empty root", root: "", bucket: "bucket", key: "audio/a.flac"},
{name: "empty bucket", root: "/cache", bucket: "", key: "audio/a.flac"},
{name: "bucket slash", root: "/cache", bucket: "bad/bucket", key: "audio/a.flac"},
{name: "empty key", root: "/cache", bucket: "bucket", key: ""},
{name: "escaping key", root: "/cache", bucket: "bucket", key: "../audio/a.flac"},
{name: "absolute key", root: "/cache", bucket: "bucket", key: "/audio/a.flac"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got, err := S3AudioCachePath(tt.root, tt.bucket, tt.key); err == nil {
t.Fatalf("S3AudioCachePath() = %q, want error", got)
}
})
}
}

View File

@@ -0,0 +1,112 @@
package artifacts
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// PreviousArtifactRequirement describes one previous-session artifact dependency.
type PreviousArtifactRequirement struct {
Name string
Required bool
Sources []string
}
// CollectPreviousArtifactRequirements scans enabled Scriptorium artifacts and returns
// deduplicated previous-session artifact requirements in deterministic order.
func CollectPreviousArtifactRequirements(
artifactsCfg map[string]config.ScriptoriumArtifactConfig,
) []PreviousArtifactRequirement {
if len(artifactsCfg) == 0 {
return nil
}
artifactNames := sortedScriptoriumArtifactNames(artifactsCfg)
byName := map[string]PreviousArtifactRequirement{}
for _, artifactName := range artifactNames {
artifactCfg := artifactsCfg[artifactName]
if !artifactCfg.Enabled {
continue
}
inputNames := sortedScriptoriumInputKeys(artifactCfg.Inputs)
for _, inputName := range inputNames {
inputCfg := artifactCfg.Inputs[inputName]
previousName, ok := PreviousSessionArtifactName(inputCfg.Source)
if !ok {
continue
}
location := fmt.Sprintf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source",
artifactName,
inputName,
)
requirement := byName[previousName]
requirement.Name = previousName
requirement.Required = requirement.Required || inputCfg.Required
requirement.Sources = append(requirement.Sources, location)
byName[previousName] = requirement
}
}
if len(byName) == 0 {
return nil
}
requirements := make([]PreviousArtifactRequirement, 0, len(byName))
for _, requirement := range byName {
requirement.Sources = dedupeAndSortStrings(requirement.Sources)
requirements = append(requirements, requirement)
}
sort.Slice(requirements, func(i, j int) bool {
return requirements[i].Name < requirements[j].Name
})
return requirements
}
func sortedScriptoriumArtifactNames(artifactsCfg map[string]config.ScriptoriumArtifactConfig) []string {
names := make([]string, 0, len(artifactsCfg))
for name := range artifactsCfg {
names = append(names, name)
}
sort.Strings(names)
return names
}
func sortedScriptoriumInputKeys(inputs map[string]config.ScriptoriumInputConfig) []string {
if len(inputs) == 0 {
return nil
}
names := make([]string, 0, len(inputs))
for name := range inputs {
names = append(names, name)
}
sort.Strings(names)
return names
}
func dedupeAndSortStrings(values []string) []string {
if len(values) == 0 {
return nil
}
seen := map[string]struct{}{}
out := make([]string, 0, len(values))
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
continue
}
if _, ok := seen[trimmed]; ok {
continue
}
seen[trimmed] = struct{}{}
out = append(out, trimmed)
}
sort.Strings(out)
return out
}

View File

@@ -0,0 +1,190 @@
package artifacts
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestCollectPreviousArtifactRequirements(t *testing.T) {
tests := []struct {
name string
artifactsCfg map[string]config.ScriptoriumArtifactConfig
want []PreviousArtifactRequirement
}{
{
name: "no artifacts",
artifactsCfg: nil,
want: nil,
},
{
name: "no previous inputs",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"session_recap": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"transcript": {Source: "narratio.transcript.trimmed", Required: true},
},
},
},
want: nil,
},
{
name: "one optional previous input",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"session_recap": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"previous_recap": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "session_recap",
Required: false,
Sources: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source"},
},
},
},
{
name: "one required previous input",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"quest_log": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"previous_quest_log": {Source: "narratio.previous_session.artifact.quest_log", Required: true},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "quest_log",
Required: true,
Sources: []string{"pipeline.scriptorium.artifacts.quest_log.inputs.previous_quest_log.source"},
},
},
},
{
name: "duplicate references are deduped",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"a": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
"b": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"y": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "session_recap",
Required: false,
Sources: []string{
"pipeline.scriptorium.artifacts.a.inputs.x.source",
"pipeline.scriptorium.artifacts.b.inputs.y.source",
},
},
},
},
{
name: "required plus optional reference becomes required",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"a": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
"b": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"y": {Source: "narratio.previous_session.artifact.session_recap", Required: true},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "session_recap",
Required: true,
Sources: []string{
"pipeline.scriptorium.artifacts.a.inputs.x.source",
"pipeline.scriptorium.artifacts.b.inputs.y.source",
},
},
},
},
{
name: "disabled artifact references are ignored",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"disabled_artifact": {
Enabled: false,
Inputs: map[string]config.ScriptoriumInputConfig{
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: true},
},
},
"enabled_artifact": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"y": {Source: "narratio.previous_session.artifact.quest_log", Required: false},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "quest_log",
Required: false,
Sources: []string{"pipeline.scriptorium.artifacts.enabled_artifact.inputs.y.source"},
},
},
},
{
name: "deterministic ordering",
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
"zz": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"b_input": {Source: "narratio.previous_session.artifact.quest_log", Required: false},
"a_input": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
"aa": {
Enabled: true,
Inputs: map[string]config.ScriptoriumInputConfig{
"c_input": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
},
},
},
want: []PreviousArtifactRequirement{
{
Name: "quest_log",
Required: false,
Sources: []string{"pipeline.scriptorium.artifacts.zz.inputs.b_input.source"},
},
{
Name: "session_recap",
Required: false,
Sources: []string{
"pipeline.scriptorium.artifacts.aa.inputs.c_input.source",
"pipeline.scriptorium.artifacts.zz.inputs.a_input.source",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := CollectPreviousArtifactRequirements(tt.artifactsCfg)
if !reflect.DeepEqual(got, tt.want) {
t.Fatalf("CollectPreviousArtifactRequirements() = %#v, want %#v", got, tt.want)
}
})
}
}

View File

@@ -34,6 +34,18 @@ func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
return ensureS3TrailingSlash(key)
}
// S3SessionConfigKey returns the session config key.
// Format: {session_prefix}/session.yml
func S3SessionConfigKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "session.yml")
}
// S3SessionLocksKey returns the mutable session lock store key.
// Format: {session_prefix}/locks.yml
func S3SessionLocksKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "locks.yml")
}
// S3CurrentManifestKey returns the current manifest pointer key.
// Format: {session_prefix}/current/manifest.json
func S3CurrentManifestKey(sessionPrefix string) string {

View File

@@ -17,6 +17,16 @@ func TestS3KeyConstruction(t *testing.T) {
t.Fatalf("audioPrefix = %q", audioPrefix)
}
sessionConfigKey := S3SessionConfigKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
if sessionConfigKey != "dnd/campaigns/forsaken/sessions/2026-04-19/session.yml" {
t.Fatalf("session config key = %q", sessionConfigKey)
}
locksKey := S3SessionLocksKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
if locksKey != "dnd/campaigns/forsaken/sessions/2026-04-19/locks.yml" {
t.Fatalf("locks key = %q", locksKey)
}
runPrefix := S3RunPrefix(sessionPrefix, runID)
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
if runPrefix != wantRunPrefix {

223
internal/audio/s3_audio.go Normal file
View File

@@ -0,0 +1,223 @@
package audio
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
// S3MaterializeRequest describes one S3-backed audio materialization.
type S3MaterializeRequest struct {
Store storage.ObjectStore
Object storage.ObjectInfo
Bucket string
CacheRoot string
CacheEnabled bool
SpoolPath string
DestPath string
}
// S3MaterializeResult captures local materialization provenance.
type S3MaterializeResult struct {
Checksum string
CachePath string
SpoolPath string
CacheHit bool
Downloaded bool
}
// MaterializeS3Audio installs one S3 audio object into the destination path,
// reusing and refreshing the durable local cache when enabled.
func MaterializeS3Audio(ctx context.Context, req S3MaterializeRequest) (S3MaterializeResult, error) {
if err := ctx.Err(); err != nil {
return S3MaterializeResult{}, err
}
if req.Store == nil {
return S3MaterializeResult{}, fmt.Errorf("object store is required")
}
key := strings.TrimSpace(req.Object.Key)
if key == "" {
return S3MaterializeResult{}, fmt.Errorf("s3 object key is required")
}
if strings.TrimSpace(req.DestPath) == "" {
return S3MaterializeResult{}, fmt.Errorf("destination path is required")
}
result := S3MaterializeResult{}
if req.CacheEnabled {
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("resolve audio cache path: %w", err)
}
result.CachePath = cachePath
if ok, err := validCachedAudio(cachePath, req.Object.Size); err != nil {
return S3MaterializeResult{}, err
} else if ok {
checksum, err := copyFileAtomicWithChecksum(cachePath, req.DestPath, 0o644)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("materialize cached audio %q: %w", cachePath, err)
}
result.Checksum = checksum
result.CacheHit = true
return result, nil
}
}
spoolPath := strings.TrimSpace(req.SpoolPath)
if spoolPath == "" {
return S3MaterializeResult{}, fmt.Errorf("spool path is required for s3 audio download")
}
if err := downloadObjectAtomic(ctx, req.Store, key, spoolPath); err != nil {
return S3MaterializeResult{}, fmt.Errorf("download s3 audio object %q: %w", key, err)
}
if err := validateLocalAudio(spoolPath, req.Object.Size); err != nil {
return S3MaterializeResult{}, fmt.Errorf("validate downloaded audio %q: %w", spoolPath, err)
}
checksum, err := copyFileAtomicWithChecksum(spoolPath, req.DestPath, 0o644)
if err != nil {
return S3MaterializeResult{}, fmt.Errorf("materialize downloaded audio %q: %w", filepath.Base(req.DestPath), err)
}
result.Checksum = checksum
result.SpoolPath = spoolPath
result.Downloaded = true
if result.CachePath != "" {
if _, err := copyFileAtomicWithChecksum(spoolPath, result.CachePath, 0o644); err != nil {
return S3MaterializeResult{}, fmt.Errorf("populate audio cache %q: %w", result.CachePath, err)
}
}
return result, nil
}
func validCachedAudio(path string, expectedSize int64) (bool, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("stat cached audio %q: %w", path, err)
}
if info.IsDir() {
return false, fmt.Errorf("cached audio path is a directory: %q", path)
}
if info.Size() <= 0 {
return false, nil
}
if expectedSize > 0 && info.Size() != expectedSize {
return false, nil
}
return true, nil
}
func validateLocalAudio(path string, expectedSize int64) error {
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("stat: %w", err)
}
if info.IsDir() {
return fmt.Errorf("path is a directory")
}
if info.Size() <= 0 {
return fmt.Errorf("file is empty")
}
if expectedSize > 0 && info.Size() != expectedSize {
return fmt.Errorf("size %d does not match remote size %d", info.Size(), expectedSize)
}
return nil
}
func downloadObjectAtomic(ctx context.Context, store storage.ObjectStore, key, destPath string) error {
if strings.TrimSpace(destPath) == "" {
return fmt.Errorf("destination path is required")
}
dir := filepath.Dir(destPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create destination directory: %w", err)
}
base := filepath.Base(destPath)
tmp, err := os.CreateTemp(dir, "."+base+".download-*.tmp")
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
if err := tmp.Close(); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("close temp file: %w", err)
}
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
if err := store.Download(ctx, key, tmpPath); err != nil {
return err
}
if err := os.Chmod(tmpPath, 0o644); err != nil {
return fmt.Errorf("set temp file permissions: %w", err)
}
if err := os.Rename(tmpPath, destPath); err != nil {
return fmt.Errorf("install downloaded file: %w", err)
}
removeTmp = false
return nil
}
func copyFileAtomicWithChecksum(src, dst string, perm os.FileMode) (string, error) {
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
return "", fmt.Errorf("source and destination paths are required")
}
in, err := os.Open(src)
if err != nil {
return "", err
}
defer func() { _ = in.Close() }()
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return "", fmt.Errorf("create destination directory: %w", err)
}
base := filepath.Base(dst)
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+base+".tmp-*")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
digest := sha256.New()
if _, err := io.Copy(io.MultiWriter(tmp, digest), in); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("copy file: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("sync temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp file: %w", err)
}
if err := os.Chmod(tmpPath, perm); err != nil {
return "", fmt.Errorf("chmod temp file: %w", err)
}
if err := os.Rename(tmpPath, dst); err != nil {
return "", fmt.Errorf("install temp file: %w", err)
}
removeTmp = false
return hex.EncodeToString(digest.Sum(nil)), nil
}

View File

@@ -0,0 +1,145 @@
package audio
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func TestMaterializeS3AudioCacheMissDownloadsAndPopulatesCache(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit {
t.Fatalf("result = %#v, want downloaded miss", result)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio")
assertFileEquals(t, req.SpoolPath, "audio")
assertFileEquals(t, result.CachePath, "audio")
if result.Checksum == "" {
t.Fatalf("checksum is empty")
}
}
func TestMaterializeS3AudioCacheHitSkipsDownload(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
t.Fatalf("cache path: %v", err)
}
writeAudioTestFile(t, cachePath, "audio")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.CacheHit || result.Downloaded {
t.Fatalf("result = %#v, want cache hit", result)
}
if len(fake.Downloads) != 0 {
t.Fatalf("downloads = %d, want 0", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "audio")
if result.SpoolPath != "" {
t.Fatalf("spool path = %q, want empty on cache hit", result.SpoolPath)
}
}
func TestMaterializeS3AudioInvalidCacheRefreshesFromS3(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("fresh-audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("fresh-audio")))
cachePath, err := artifacts.S3AudioCachePath(req.CacheRoot, req.Bucket, key)
if err != nil {
t.Fatalf("cache path: %v", err)
}
writeAudioTestFile(t, cachePath, "stale")
result, err := MaterializeS3Audio(context.Background(), req)
if err != nil {
t.Fatalf("MaterializeS3Audio() error = %v", err)
}
if !result.Downloaded || result.CacheHit {
t.Fatalf("result = %#v, want refreshed miss", result)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
assertFileEquals(t, req.DestPath, "fresh-audio")
assertFileEquals(t, cachePath, "fresh-audio")
}
func TestMaterializeS3AudioDownloadFailureLeavesDestinationMissing(t *testing.T) {
root := t.TempDir()
fake := &storage.FakeBackend{DownloadErr: os.ErrPermission}
key := "dnd/campaigns/forsaken/sessions/2026-04-19/audio/alice.flac"
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("audio")})
req := testMaterializeRequest(t, root, fake, key, int64(len("audio")))
_, err := MaterializeS3Audio(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "download s3 audio object") {
t.Fatalf("error = %v, want download error", err)
}
assertMissing(t, req.DestPath)
}
func testMaterializeRequest(t *testing.T, root string, fake *storage.FakeBackend, key string, size int64) S3MaterializeRequest {
t.Helper()
return S3MaterializeRequest{
Store: fake,
Object: storage.ObjectInfo{Key: key, Size: size, ETag: "etag"},
Bucket: "my-dnd-archive",
CacheRoot: filepath.Join(root, "cache"),
CacheEnabled: true,
SpoolPath: filepath.Join(root, "spool", "alice.flac"),
DestPath: filepath.Join(root, "work", "audio", "alice.flac"),
}
}
func writeAudioTestFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("mkdir %q: %v", path, err)
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatalf("write %q: %v", path, err)
}
}
func assertFileEquals(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
if string(data) != want {
t.Fatalf("%q = %q, want %q", path, string(data), want)
}
}
func assertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("stat %q = %v, want not exists", path, err)
}
}

View File

@@ -0,0 +1,67 @@
package config
import (
"strings"
"testing"
)
func TestCacheDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://example.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 10s
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Cache.Root != DefaultCacheRoot {
t.Fatalf("cache.root = %q, want %q", cfg.Pipeline.Cache.Root, DefaultCacheRoot)
}
if cfg.Pipeline.Cache.S3Audio == nil || *cfg.Pipeline.Cache.S3Audio != DefaultCacheS3Audio {
t.Fatalf("cache.s3_audio = %v, want %v", cfg.Pipeline.Cache.S3Audio, DefaultCacheS3Audio)
}
}
func TestCacheStrictDecodeRejectsUnknownFields(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
cache:
root: /var/cache/narratio
unknown: true
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestCacheValidationRequiresRootWhenS3AudioEnabled(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, `workspace:
root: /tmp/narratio
cache:
root: " "
s3_audio: true
`, `session_id: 2026-05-03
inputs:
audio_dir: ./audio
`)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.cache.root is required") {
t.Fatalf("Validate() error = %v, want cache root error", err)
}
}

View File

@@ -0,0 +1,137 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestCampaignConfigDefaultSearchOrder(t *testing.T) {
want := []string{
"/usr/local/etc/narratio/campaign.yml",
"/etc/narratio/campaign.yml",
}
if len(DefaultCampaignConfigSearchPaths) != len(want) {
t.Fatalf("DefaultCampaignConfigSearchPaths = %#v, want %#v", DefaultCampaignConfigSearchPaths, want)
}
for i := range want {
if DefaultCampaignConfigSearchPaths[i] != want[i] {
t.Fatalf("DefaultCampaignConfigSearchPaths[%d] = %q, want %q", i, DefaultCampaignConfigSearchPaths[i], want[i])
}
}
}
func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\nunknown: true\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
}
}
func TestCampaignSessionMergeFillsStableInputs(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
if cfg.Session.Campaign != "sample-campaign" {
t.Fatalf("session campaign = %q, want campaign config value", cfg.Session.Campaign)
}
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./campaign-speakers.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
}
func TestCampaignSessionMergeSessionOverridesStableInputs(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
"session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n speakers_file: ./session-speakers.yml\n",
)
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadWithSessionOptions() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./session-speakers.yml", sessionPath, "session_config")
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
}
func TestCampaignSessionMismatchFails(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ncampaign: other-campaign\ninputs:\n audio_dir: ./audio\n",
)
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "does not match campaign config") {
t.Fatalf("error = %q, want campaign mismatch context", err.Error())
}
}
func TestLoadMissingCampaignFileFails(t *testing.T) {
pipelinePath, _, sessionPath := writeCampaignConfigTestFiles(t,
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
)
missingCampaignPath := filepath.Join(filepath.Dir(sessionPath), "missing-campaign.yml")
_, err := LoadWithSessionOptions(pipelinePath, missingCampaignPath, sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected load error, got nil")
}
if !strings.Contains(err.Error(), "load campaign config") {
t.Fatalf("error = %q, want campaign load context", err.Error())
}
}
func writeCampaignConfigTestFiles(t *testing.T, campaignYAML, sessionYAML string) (string, string, string) {
t.Helper()
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := "workspace:\n root: " + filepath.ToSlash(filepath.Join(dir, "work")) + "\nwhisperx:\n transcribe_url: https://example.com/transcribe\nanalyzer:\n timeout: 20m\nnotification:\n timeout: 10s\n"
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
return pipelinePath, campaignPath, sessionPath
}
func assertResolvedStableInput(t *testing.T, got ResolvedInputFile, wantPath, wantConfigPath, wantSource string) {
t.Helper()
if got.Path != wantPath || got.ConfigPath != wantConfigPath || got.Source != wantSource {
t.Fatalf("resolved input = %#v, want path=%q config_path=%q source=%q", got, wantPath, wantConfigPath, wantSource)
}
}

View File

@@ -1,11 +1,17 @@
package config
// Config is the resolved combined configuration from pipeline.yml and session.yml.
// Config is the resolved combined configuration from pipeline.yml,
// campaign.yml, and session.yml.
type Config struct {
Pipeline *PipelineConfig
Campaign *CampaignConfig
Session *SessionConfig
PipelinePath string
CampaignPath string
SessionPath string
StableInputs ResolvedStableInputs
SessionSource SessionSource
}
// PipelineConfig contains durable pipeline-level settings.
@@ -13,6 +19,7 @@ type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"`
Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"`
Cache CacheConfig `yaml:"cache"`
Archive *ArchiveConfig `yaml:"archive"`
Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"`
@@ -25,13 +32,27 @@ type PipelineConfig struct {
Notification NotificationConfig `yaml:"notification"`
}
// CampaignConfig contains stable campaign-level identity and input defaults.
type CampaignConfig struct {
Campaign string `yaml:"campaign"`
Inputs CampaignInputsConfig `yaml:"inputs"`
}
// CampaignInputsConfig contains stable campaign-level input file references.
type CampaignInputsConfig struct {
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
}
// SessionConfig contains per-session inputs and metadata.
type SessionConfig struct {
SessionID string `yaml:"session_id"`
Campaign string `yaml:"campaign"`
Date string `yaml:"date"`
Title string `yaml:"title"`
Inputs SessionInputsConfig `yaml:"inputs"`
SessionID string `yaml:"session_id"`
PreviousSessionID string `yaml:"previous_session_id"`
Campaign string `yaml:"campaign"`
Date string `yaml:"date"`
Title string `yaml:"title"`
Inputs SessionInputsConfig `yaml:"inputs"`
}
// WorkspaceConfig configures local workspace behavior.
@@ -70,11 +91,18 @@ type SpoolConfig struct {
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
}
// CacheConfig configures durable local caches for reusable remote inputs.
type CacheConfig struct {
Root string `yaml:"root"`
S3Audio *bool `yaml:"s3_audio"`
}
// ArchiveConfig configures archive behavior and artifact promotions.
type ArchiveConfig struct {
Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"`
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
Locks []ArchiveLockRule `yaml:"locks"`
}
// ArchivePromotionRule configures one artifact promotion mapping.
@@ -84,6 +112,18 @@ type ArchivePromotionRule struct {
Required *bool `yaml:"required"`
}
// ArchiveLockRule prevents one source-based promotion from overwriting its
// top-level archive destination.
type ArchiveLockRule struct {
Source string `yaml:"source"`
Reason string `yaml:"reason"`
}
// ArchiveLockStore is the mutable per-session remote lock store.
type ArchiveLockStore struct {
Locks []ArchiveLockRule `yaml:"locks"`
}
// WhisperXConfig configures WhisperX adapter settings.
type WhisperXConfig struct {
TranscribeURL string `yaml:"transcribe_url"`
@@ -228,3 +268,29 @@ type SessionInputsConfig struct {
type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"`
}
// ResolvedStableInputs records where stable input file paths came from after
// campaign/session merge.
type ResolvedStableInputs struct {
SpeakersFile ResolvedInputFile
AutocorrectFile ResolvedInputFile
GlossaryFile ResolvedInputFile
}
// ResolvedInputFile records one merged config path and its source config file.
type ResolvedInputFile struct {
Path string
ConfigPath string
Source string
}
// SessionSource records where session.yml came from before materialization.
type SessionSource struct {
Source string
LocalPath string
S3Bucket string
S3Key string
S3Size int64
S3ETag string
SpoolPath string
}

View File

@@ -1,11 +1,12 @@
package config
// Default filesystem locations for pipeline configuration lookup when --config
// is omitted. Order is highest to lowest precedence.
// Default filesystem locations for config lookup when config path flags are
// omitted. Order is highest to lowest precedence.
const (
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
DefaultSessionConfigPathLocal = "./session.yml"
DefaultCampaignConfigPathUsrLocal = "/usr/local/etc/narratio/campaign.yml"
DefaultCampaignConfigPathEtc = "/etc/narratio/campaign.yml"
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
@@ -13,6 +14,8 @@ const (
DefaultStorageS3RootPrefix = "dnd"
DefaultWorkspaceRoot = "/var/lib/narratio"
DefaultSpoolRoot = "/var/spool/narratio"
DefaultCacheRoot = "/var/cache/narratio"
DefaultCacheS3Audio = true
DefaultWhisperXLanguage = "en"
DefaultWhisperXTimeout = "30m"
@@ -56,6 +59,7 @@ const (
PathLogsDirSegment = "logs"
PathCurrentDirSegment = "current"
PathRunsDirSegment = "runs"
PathPreviousDirSegment = "previous"
PathManifestFile = "manifest.json"
PathLockFile = ".lock"
PathTranscriptMerged = "transcripts/merged.json"
@@ -87,13 +91,22 @@ var DefaultPipelineConfigSearchPaths = []string{
DefaultPipelineConfigPathEtc,
}
// DefaultCampaignConfigSearchPaths defines the default search order for
// campaign.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultCampaignConfigSearchPaths = []string{
DefaultCampaignConfigPathUsrLocal,
DefaultCampaignConfigPathEtc,
}
// DefaultSessionConfigSearchPaths defines the default search order for
// session.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultSessionConfigSearchPaths = []string{
DefaultSessionConfigPathLocal,
DefaultSessionConfigPathUsrLocal,
DefaultSessionConfigPathEtc,
}

View File

@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"gopkg.in/yaml.v3"
@@ -21,6 +22,15 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return &cfg, nil
}
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
func LoadCampaign(path string) (*CampaignConfig, error) {
var cfg CampaignConfig
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
return nil, fmt.Errorf("load campaign config: %w", err)
}
return &cfg, nil
}
// LoadSession loads session configuration from a YAML file with strict field checking.
func LoadSession(path string) (*SessionConfig, error) {
return LoadSessionWithOptions(path, SessionLoadOptions{})
@@ -28,7 +38,8 @@ func LoadSession(path string) (*SessionConfig, error) {
// SessionLoadOptions configures session template rendering behavior.
type SessionLoadOptions struct {
SessionID string
SessionID string
PreviousSessionID string
}
// LoadSessionWithOptions loads session configuration from a YAML file with
@@ -38,53 +49,207 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
if err != nil {
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
}
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
}
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
// strict field checking after template rendering.
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
rendered, err := renderSessionTemplate(string(data), opts)
if err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
var cfg SessionConfig
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
path,
label,
strings.TrimSpace(opts.SessionID),
strings.TrimSpace(cfg.SessionID),
)
}
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
label,
strings.TrimSpace(opts.PreviousSessionID),
strings.TrimSpace(cfg.PreviousSessionID),
)
}
return &cfg, nil
}
// Load loads and resolves combined pipeline and session configuration.
func Load(pipelinePath, sessionPath string) (*Config, error) {
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
// LoadArchiveLockStoreBytes loads a mutable session lock store with strict
// field checking and source validation.
func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
var store ArchiveLockStore
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
return nil, fmt.Errorf("load archive lock store: %w", err)
}
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
if err != nil {
return nil, fmt.Errorf("load archive lock store: %w", err)
}
store.Locks = locks
return &store, nil
}
// LoadWithSessionOptions loads and resolves combined pipeline and session
// configuration with session template options.
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
// MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
if store == nil {
store = &ArchiveLockStore{}
}
data, err := yaml.Marshal(store)
if err != nil {
return nil, fmt.Errorf("marshal archive lock store: %w", err)
}
return data, nil
}
// Load loads and resolves combined pipeline, campaign, and session configuration.
// Passing only a session path is supported for package-internal compatibility;
// in that form campaign.yml is expected next to the session file.
func Load(pipelinePath string, paths ...string) (*Config, error) {
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
if err != nil {
return nil, err
}
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
}
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
// session configuration with session template options.
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipeline(pipelinePath)
if err != nil {
return nil, err
}
campaignCfg, err := LoadCampaign(campaignPath)
if err != nil {
return nil, err
}
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
if err != nil {
return nil, err
}
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
Source: "session_config",
LocalPath: sessionPath,
})
}
// Resolve builds final stage-facing configuration from already loaded
// pipeline, campaign, and session documents.
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
if err != nil {
return nil, err
}
if strings.TrimSpace(sessionSource.Source) == "" {
sessionSource.Source = "session_config"
}
if strings.TrimSpace(sessionSource.LocalPath) == "" {
sessionSource.LocalPath = sessionPath
}
return &Config{
Pipeline: pipelineCfg,
Session: sessionCfg,
PipelinePath: pipelinePath,
SessionPath: sessionPath,
Pipeline: pipelineCfg,
Campaign: campaignCfg,
Session: sessionCfg,
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
StableInputs: stableInputs,
SessionSource: sessionSource,
}, nil
}
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
switch len(paths) {
case 1:
sessionPath = paths[0]
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
case 2:
campaignPath = paths[0]
sessionPath = paths[1]
default:
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
}
return campaignPath, sessionPath, nil
}
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
if campaignCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
}
if sessionCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
}
campaignName := strings.TrimSpace(campaignCfg.Campaign)
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
return ResolvedStableInputs{}, fmt.Errorf(
"campaign/session config invalid: session campaign %q does not match campaign config %q",
sessionCampaign,
campaignName,
)
}
if sessionCampaign == "" {
sessionCfg.Campaign = campaignName
}
stable := ResolvedStableInputs{
SpeakersFile: selectStableInput(
campaignCfg.Inputs.SpeakersFile,
sessionCfg.Inputs.SpeakersFile,
campaignPath,
sessionPath,
),
AutocorrectFile: selectStableInput(
campaignCfg.Inputs.AutocorrectFile,
sessionCfg.Inputs.AutocorrectFile,
campaignPath,
sessionPath,
),
GlossaryFile: selectStableInput(
campaignCfg.Inputs.GlossaryFile,
sessionCfg.Inputs.GlossaryFile,
campaignPath,
sessionPath,
),
}
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
return stable, nil
}
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
if strings.TrimSpace(sessionValue) != "" {
return ResolvedInputFile{
Path: sessionValue,
ConfigPath: sessionPath,
Source: "session_config",
}
}
return ResolvedInputFile{
Path: campaignValue,
ConfigPath: campaignPath,
Source: "campaign_config",
}
}
func decodeStrictYAML(kind, path string, out any) error {
f, err := os.Open(path)
if err != nil {
@@ -114,24 +279,36 @@ var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
sessionID := strings.TrimSpace(opts.SessionID)
previousSessionID := strings.TrimSpace(opts.PreviousSessionID)
rendered := content
if sessionID != "" {
rendered = strings.ReplaceAll(rendered, "{{session_id}}", sessionID)
rendered = strings.ReplaceAll(rendered, "{{ session_id }}", sessionID)
rendered = replaceTemplateVariable(rendered, "session_id", sessionID)
}
if previousSessionID != "" {
rendered = replaceTemplateVariable(rendered, "previous_session_id", previousSessionID)
}
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
if len(unresolved) > 0 {
seenVars := map[string]struct{}{}
vars := make([]string, 0, len(unresolved))
for _, m := range unresolved {
if len(m) > 1 {
vars = append(vars, m[1])
name := m[1]
if _, ok := seenVars[name]; ok {
continue
}
seenVars[name] = struct{}{}
vars = append(vars, name)
}
}
sort.Strings(vars)
if len(vars) > 0 {
hints := unresolvedTemplateHints(vars)
return "", fmt.Errorf(
"session file template rendering failed: unresolved template variable(s): %s; pass --session-id when using {{ session_id }}",
"session file template rendering failed: unresolved template variable(s): %s%s",
strings.Join(vars, ", "),
hints,
)
}
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
@@ -140,6 +317,35 @@ func renderSessionTemplate(content string, opts SessionLoadOptions) (string, err
return rendered, nil
}
func replaceTemplateVariable(content, name, value string) string {
rendered := strings.ReplaceAll(content, "{{"+name+"}}", value)
rendered = strings.ReplaceAll(rendered, "{{ "+name+" }}", value)
return rendered
}
func unresolvedTemplateHints(vars []string) string {
seen := map[string]struct{}{}
flags := make([]string, 0, 2)
for _, name := range vars {
switch name {
case "session_id":
if _, ok := seen["--session-id"]; !ok {
seen["--session-id"] = struct{}{}
flags = append(flags, "--session-id")
}
case "previous_session_id":
if _, ok := seen["--previous-session-id"]; !ok {
seen["--previous-session-id"] = struct{}{}
flags = append(flags, "--previous-session-id")
}
}
}
if len(flags) == 0 {
return ""
}
return "; pass " + strings.Join(flags, " and ") + " when using those template variable(s)"
}
func shortName(path, fallback string) string {
base := filepath.Base(path)
if base == "." || base == string(filepath.Separator) {
@@ -155,6 +361,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
applyWorkspaceDefaults(&cfg.Workspace)
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyArchiveDefaults(&cfg.Archive)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
@@ -203,6 +410,18 @@ func applySpoolDefaults(cfg *SpoolConfig) {
}
}
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyArchiveDefaults(cfg **ArchiveConfig) {
if cfg == nil {
return

View File

@@ -187,6 +187,43 @@ inputs:
`,
wantValidate: "session config \"session.yml\" invalid: session.session_id is required",
},
{
name: "valid previous_session_id passes",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
`,
sessionYAML: `session_id: 2026-05-03
previous_session_id: 2026-04-26
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
},
{
name: "previous_session_id equal to session_id fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
`,
sessionYAML: `session_id: 2026-05-03
previous_session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "session config \"session.yml\" invalid: session.previous_session_id must not equal session.session_id",
},
{
name: "missing transcribe_url fails",
pipelineYAML: `workspace:
@@ -867,6 +904,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
Report: boolPtr(true),
},
},
Campaign: &CampaignConfig{Campaign: "sample-campaign"},
Session: &SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
@@ -926,6 +964,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
campaignPath := filepath.Join(examplesDir, "campaign.yml")
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
var (
@@ -935,7 +974,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
cfg, err = Load(pipelinePath, sessionPath)
} else {
cfg, err = LoadWithSessionOptions(pipelinePath, sessionPath, tt.sessionOpts)
cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, tt.sessionOpts)
}
if err != nil {
t.Fatalf("load example config error = %v", err)
@@ -964,14 +1003,34 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
campaignPath := filepath.Join(dir, "campaign.yml")
sessionPath := filepath.Join(dir, "session.yml")
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
campaignYAML := `campaign: ` + campaignNameFromSessionYAML(sessionYAML) + `
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
t.Fatalf("write campaign.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
return pipelinePath, sessionPath
}
func campaignNameFromSessionYAML(sessionYAML string) string {
for _, line := range strings.Split(sessionYAML, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "campaign:") {
return strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "campaign:")), `"'`)
}
}
return "sample-campaign"
}

View File

@@ -116,6 +116,75 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
output_kind: session_recap
`,
},
{
name: "canonical previous-session source is accepted",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
transcript:
source: narratio.transcript.polished
required: true
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
vars:
session_id: true
output_kind: session_recap
`,
},
{
name: "canonical previous-session source missing artifact key fails validation",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
previous_recap:
source: narratio.previous_session.artifact.
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact." must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
},
{
name: "canonical previous-session source invalid artifact key fails validation",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
previous_recap:
source: narratio.previous_session.artifact.session-recap
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.session-recap" must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
},
{
name: "canonical previous-session source unknown artifact fails validation",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
previous_recap:
source: narratio.previous_session.artifact.quest_log
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.quest_log" references unknown artifact "quest_log"`,
},
{
name: "canonical artifact source is accepted",
scriptoriumYAML: `scriptorium:

View File

@@ -55,6 +55,34 @@ inputs:
}
}
func TestLoadSessionWithOptionsRendersPreviousSessionPlaceholder(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{
SessionID: "2026-04-04",
PreviousSessionID: "2026-03-28",
})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.PreviousSessionID != "2026-03-28" {
t.Fatalf("PreviousSessionID = %q, want 2026-03-28", cfg.PreviousSessionID)
}
}
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
@@ -82,6 +110,37 @@ inputs:
}
}
func TestLoadSessionWithOptionsUnresolvedPreviousSessionPlaceholderFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
previous_session_id: "{{ previous_session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "unresolved template variable") {
t.Fatalf("error = %q, want unresolved-variable context", err.Error())
}
if !strings.Contains(err.Error(), "previous_session_id") {
t.Fatalf("error = %q, want previous_session_id variable", err.Error())
}
if !strings.Contains(err.Error(), "--previous-session-id") {
t.Fatalf("error = %q, want previous-session-id guidance", err.Error())
}
}
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
@@ -106,6 +165,34 @@ inputs:
}
}
func TestLoadSessionWithOptionsPreviousSessionMismatchFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
previous_session_id: 2026-04-26
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{
SessionID: "2026-05-03",
PreviousSessionID: "2026-04-25",
})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "previous_session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
@@ -154,3 +241,38 @@ inputs:
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
}
}
func TestLoadSessionBytesWithOptionsUsesSameTemplateAndStrictDecode(t *testing.T) {
sessionYAML := []byte(`session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_s3:
prefix: audio/
`)
cfg, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", sessionYAML, SessionLoadOptions{SessionID: "2026-05-03"})
if err != nil {
t.Fatalf("LoadSessionBytesWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
}
_, err = LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\nunknown: true\n"), SessionLoadOptions{})
if err == nil {
t.Fatal("expected strict decode error, got nil")
}
if !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want strict decode context", err.Error())
}
}
func TestLoadSessionBytesWithOptionsMismatchFails(t *testing.T) {
_, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\n"), SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected mismatch error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}

View File

@@ -287,6 +287,100 @@ archive:
}
}
func TestArchiveLockValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "valid built in source",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
reason: reviewed transcript
`,
},
{
name: "valid configured source",
pipelineYML: testPipelineBaseYAML + `
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
archive:
locks:
- source: narratio.artifact.session_recap
`,
},
{
name: "missing source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- reason: no source
`,
wantErr: "pipeline.archive.locks[0].source is required",
},
{
name: "invalid source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.unknown
`,
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
},
{
name: "duplicate source rejected",
pipelineYML: testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
- source: " narratio.transcript.trimmed "
`,
wantErr: "duplicates another archive lock source",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
archive:
locks:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
archive:
@@ -301,6 +395,54 @@ archive:
}
}
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
store, err := LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
reason: reviewed
`), nil)
if err != nil {
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
}
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.trimmed" || store.Locks[0].Reason != "reviewed" {
t.Fatalf("locks = %#v", store.Locks)
}
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
dest: transcripts/trimmed.json
`), nil)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("unknown field error = %v, want strict decode failed", err)
}
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.trimmed
- source: narratio.transcript.trimmed
`), nil)
if err == nil || !strings.Contains(err.Error(), "duplicates another archive lock source") {
t.Fatalf("duplicate error = %v", err)
}
}
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
merged := MergeArchiveLockRules(
[]ArchiveLockRule{{Source: "narratio.transcript.trimmed", Reason: "static"}},
[]ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "remote"},
{Source: "narratio.transcript.full", Reason: "remote full"},
},
)
if len(merged) != 2 {
t.Fatalf("merged len = %d, want 2: %#v", len(merged), merged)
}
if merged[0].Source != "narratio.transcript.trimmed" || merged[0].Reason != "static" {
t.Fatalf("merged[0] = %#v, want static lock", merged[0])
}
if merged[1].Source != "narratio.transcript.full" {
t.Fatalf("merged[1] = %#v, want remote full lock", merged[1])
}
}
func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct {
name string

View File

@@ -17,6 +17,9 @@ func Validate(cfg *Config) error {
if cfg.Pipeline == nil {
return fmt.Errorf("pipeline config is required")
}
if cfg.Campaign == nil {
return fmt.Errorf("campaign config is required")
}
if cfg.Session == nil {
return fmt.Errorf("session config is required")
}
@@ -24,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validatePipeline(cfg.Pipeline); err != nil {
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
}
if err := validateCampaign(cfg.Campaign); err != nil {
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
}
if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
}
@@ -34,6 +40,16 @@ func Validate(cfg *Config) error {
return nil
}
func validateCampaign(cfg *CampaignConfig) error {
if cfg == nil {
return fmt.Errorf("campaign config is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("campaign.campaign is required")
}
return nil
}
func validatePipeline(cfg *PipelineConfig) error {
if strings.TrimSpace(cfg.Workspace.Root) == "" {
return fmt.Errorf("pipeline.workspace.root is required")
@@ -47,6 +63,9 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateSpool(cfg.Spool); err != nil {
return err
}
if err := validateCache(cfg.Cache); err != nil {
return err
}
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
return err
}
@@ -104,6 +123,13 @@ func validateSpool(cfg SpoolConfig) error {
return nil
}
func validateCache(cfg CacheConfig) error {
if cfg.S3Audio != nil && *cfg.S3Audio && strings.TrimSpace(cfg.Root) == "" {
return fmt.Errorf("pipeline.cache.root is required when pipeline.cache.s3_audio is true")
}
return nil
}
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
if cfg == nil {
return nil
@@ -136,9 +162,69 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
}
seenDest[normalizedDest] = struct{}{}
}
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
if err != nil {
return err
}
cfg.Locks = locks
return nil
}
// ValidateArchiveLockRules validates and normalizes source-based archive locks.
func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumConfig, label string) ([]ArchiveLockRule, error) {
seenLocks := map[string]struct{}{}
out := make([]ArchiveLockRule, 0, len(locks))
if strings.TrimSpace(label) == "" {
label = "archive.locks"
}
for i, item := range locks {
prefix := fmt.Sprintf("%s[%d]", label, i)
source := strings.TrimSpace(item.Source)
if source == "" {
return nil, fmt.Errorf("%s.source is required", prefix)
}
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
}
if _, ok := seenLocks[source]; ok {
return nil, fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
}
seenLocks[source] = struct{}{}
out = append(out, ArchiveLockRule{
Source: source,
Reason: strings.TrimSpace(item.Reason),
})
}
return out, nil
}
// MergeArchiveLockRules returns the union of static and remote locks. Static
// locks win when both sources contain the same lock.
func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []ArchiveLockRule {
out := make([]ArchiveLockRule, 0, len(staticLocks)+len(remoteLocks))
seen := map[string]struct{}{}
for _, item := range staticLocks {
source := strings.TrimSpace(item.Source)
if source == "" {
continue
}
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
seen[source] = struct{}{}
}
for _, item := range remoteLocks {
source := strings.TrimSpace(item.Source)
if source == "" {
continue
}
if _, ok := seen[source]; ok {
continue
}
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
seen[source] = struct{}{}
}
return out
}
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
trimmed := strings.TrimSpace(source)
switch trimmed {
@@ -487,8 +573,14 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
}
func validateSession(cfg *SessionConfig) error {
if strings.TrimSpace(cfg.SessionID) == "" {
return fmt.Errorf("session.session_id is required")
if err := validateSessionIdentifier("session.session_id", cfg.SessionID, true); err != nil {
return err
}
if err := validateSessionIdentifier("session.previous_session_id", cfg.PreviousSessionID, false); err != nil {
return err
}
if strings.TrimSpace(cfg.PreviousSessionID) != "" && strings.TrimSpace(cfg.PreviousSessionID) == strings.TrimSpace(cfg.SessionID) {
return fmt.Errorf("session.previous_session_id must not equal session.session_id")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("session.campaign is required")
@@ -525,6 +617,16 @@ func validateSession(cfg *SessionConfig) error {
return nil
}
func validateSessionIdentifier(fieldName, value string, required bool) error {
if strings.TrimSpace(value) == "" {
if required {
return fmt.Errorf("%s is required", fieldName)
}
return nil
}
return nil
}
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
if pipeline == nil || session == nil {
return nil
@@ -563,12 +665,38 @@ var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
var narratioArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
var narratioPreviousSessionArtifactSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
if isStaticSupportedScriptoriumInputSource(source) {
trimmedSource := strings.TrimSpace(source)
if isStaticSupportedScriptoriumInputSource(trimmedSource) {
return "", nil
}
matches := narratioArtifactSourceRE.FindStringSubmatch(source)
if strings.HasPrefix(trimmedSource, "narratio.previous_session.artifact") {
matches := narratioPreviousSessionArtifactSourceRE.FindStringSubmatch(trimmedSource)
if len(matches) != 2 {
return "", fmt.Errorf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$",
artifactName,
inputName,
source,
)
}
referenced := matches[1]
if _, ok := configuredArtifacts[referenced]; !ok {
return "", fmt.Errorf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q",
artifactName,
inputName,
source,
referenced,
)
}
return "", nil
}
matches := narratioArtifactSourceRE.FindStringSubmatch(trimmedSource)
if len(matches) != 2 {
return "", fmt.Errorf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
@@ -591,7 +719,7 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
}
func isStaticSupportedScriptoriumInputSource(source string) bool {
switch strings.TrimSpace(source) {
switch source {
case "previous_session_artifact":
return true
case "narratio.transcript.merged":

View File

@@ -23,6 +23,7 @@ type InputRecord struct {
S3Size int64 `json:"s3_size,omitempty"`
S3ETag string `json:"s3_etag,omitempty"`
SpoolPath string `json:"spool_path,omitempty"`
CachePath string `json:"cache_path,omitempty"`
}
// ArtifactRecord captures one produced artifact and optional remote metadata.

View File

@@ -631,6 +631,23 @@ func resolveScriptoriumInput(
runtimeCatalog *artifacts.ArtifactCatalog,
) (string, bool, *artifacts.ResolvedSessionArtifact, error) {
source := strings.TrimSpace(inputCfg.Source)
if artifacts.IsPreviousSessionArtifactSource(source) {
resolved, err := artifacts.ResolvePreviousSessionArtifactWithCatalog(paths, m, source, runtimeCatalog)
if err == nil {
copy := resolved
return resolved.Path, true, &copy, nil
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
if inputCfg.Required {
return "", false, nil, fmt.Errorf(
"required previous-session input source %q is unavailable; run narratio run-stage --force prepare",
source,
)
}
return "", false, nil, nil
}
return "", false, nil, err
}
switch source {
case "previous_session_artifact":
if strings.TrimSpace(inputCfg.Path) == "" {

View File

@@ -10,6 +10,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -717,6 +718,146 @@ func TestAnalyzeOmitsOptionalMissingConfiguredArtifactInput(t *testing.T) {
}
}
func TestAnalyzeRequiredPreviousSessionArtifactInputGuidesPrepareForce(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
Source: "narratio.previous_session.artifact.session_recap",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "run narratio run-stage --force prepare") {
t.Fatalf("error = %q, want guidance to run force prepare", err.Error())
}
}
func TestAnalyzeResolvesCanonicalPreviousSessionArtifactFromManifestInput(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
writeAnalyzeFile(t, previousPath, "previous recap\n")
m.Inputs = append(m.Inputs, manifest.InputRecord{
Kind: "previous_artifact",
Path: previousPath,
})
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
Source: "narratio.previous_session.artifact.session_recap",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RunRequests) != 1 {
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
}
if got := fake.RunRequests[0].InputPaths["previous_recap"]; got != previousPath {
t.Fatalf("previous_recap input = %q, want %q", got, previousPath)
}
}
func TestAnalyzeOmitsOptionalMissingCanonicalPreviousSessionArtifact(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
Source: "narratio.previous_session.artifact.session_recap",
Required: false,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RunRequests) != 1 {
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
}
if _, exists := fake.RunRequests[0].InputPaths["previous_recap"]; exists {
t.Fatalf("optional canonical previous_recap should be omitted when unavailable")
}
}
func TestAnalyzeRenderDebugWithCanonicalPreviousSessionInput(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
writeAnalyzeFile(t, previousPath, "previous recap\n")
m.Inputs = append(m.Inputs, manifest.InputRecord{
Kind: "previous_artifact",
Path: previousPath,
})
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
Source: "narratio.previous_session.artifact.session_recap",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RenderRequests) != 1 || len(fake.RunRequests) != 1 {
t.Fatalf("render/run requests = %d/%d, want 1/1", len(fake.RenderRequests), len(fake.RunRequests))
}
if got := fake.RenderRequests[0].InputPaths["previous_recap"]; got != previousPath {
t.Fatalf("render previous_recap input = %q, want %q", got, previousPath)
}
if got := fake.RunRequests[0].InputPaths["previous_recap"]; got != previousPath {
t.Fatalf("run previous_recap input = %q, want %q", got, previousPath)
}
}
func TestAnalyzeDoesNotCallObjectStoreForCanonicalPreviousSessionInput(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
writeAnalyzeFile(t, previousPath, "previous recap\n")
m.Inputs = append(m.Inputs, manifest.InputRecord{
Kind: "previous_artifact",
Path: previousPath,
})
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
Source: "narratio.previous_session.artifact.session_recap",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
tracker := &analyzeObjectStoreTracker{}
env.ObjectStore = tracker
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if tracker.called {
t.Fatal("analyze should not call object store for previous-session input resolution")
}
}
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
@@ -1145,3 +1286,27 @@ func mustArtifactEntryList(t *testing.T, metadata map[string]any, key string) []
}
return out
}
type analyzeObjectStoreTracker struct {
called bool
}
func (s *analyzeObjectStoreTracker) List(context.Context, string) ([]storage.ObjectInfo, error) {
s.called = true
return nil, errors.New("unexpected object store list call")
}
func (s *analyzeObjectStoreTracker) Download(context.Context, string, string) error {
s.called = true
return errors.New("unexpected object store download call")
}
func (s *analyzeObjectStoreTracker) Upload(context.Context, string, string, storage.UploadOptions) (storage.ObjectInfo, error) {
s.called = true
return storage.ObjectInfo{}, errors.New("unexpected object store upload call")
}
func (s *analyzeObjectStoreTracker) Exists(context.Context, string) (bool, error) {
s.called = true
return false, errors.New("unexpected object store exists call")
}

View File

@@ -118,11 +118,22 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("archive: collect run files: %w", err)
}
sessionPaths := archiveSessionPaths(env, m)
previousFiles, err := collectArchivePreviousFiles(sessionPaths.PreviousDir)
if err != nil {
return nil, fmt.Errorf("archive: collect previous files: %w", err)
}
runtimeCatalog, err := buildArchiveRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
if err != nil {
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
}
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
sessionPaths,
m,
runtimeCatalog,
env.Config.Pipeline.Archive.PromoteArtifacts,
env.Config.Pipeline.Archive.Locks,
sessionPrefix,
)
if err != nil {
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
}
@@ -144,6 +155,15 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
promotedUploaded = append(promotedUploaded, promotion.Dest)
}
previousUploaded := make([]string, 0, len(previousFiles))
for _, file := range previousFiles {
key := artifacts.S3PromotedArtifactKey(sessionPrefix, file.RelativePath)
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
return nil, fmt.Errorf("archive: upload previous file %q to %q: %w", file.RelativePath, key, err)
}
previousUploaded = append(previousUploaded, file.RelativePath)
}
currentManifestKey, currentRunPointerKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
bucket,
@@ -151,7 +171,9 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
sessionPrefix,
runUploaded,
promotedUploaded,
previousUploaded,
skippedOptional,
lockedPromotions,
currentManifestKey,
))
if err != nil {
@@ -187,7 +209,11 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
"run_uploaded_paths": runUploaded,
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": promotedUploaded,
"previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": previousUploaded,
"skipped_optional_promotions": skippedOptional,
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey,
"current_run_id_key": currentRunPointerKey,
"current_pointer_written": true,
@@ -204,6 +230,16 @@ type archivePromotion struct {
Provenance string
}
type archiveLockedPromotion struct {
Source string
Dest string
RemoteKey string
Reason string
Required bool
LocalPath string
Provenance string
}
func archiveDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
@@ -303,26 +339,53 @@ func resolveArchivePromotions(
m *manifest.Manifest,
catalog *artifacts.ArtifactCatalog,
rules []config.ArchivePromotionRule,
) ([]archivePromotion, []string, error) {
locks []config.ArchiveLockRule,
sessionPrefix string,
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
out := make([]archivePromotion, 0, len(rules))
skippedOptional := make([]string, 0)
lockedPromotions := make([]archiveLockedPromotion, 0)
lockSet := archiveLockSet(locks)
for _, rule := range rules {
source := strings.TrimSpace(rule.Source)
required := rule.Required == nil || *rule.Required
dest, err := resolveArchivePromotionDest(rule, catalog)
if err != nil {
return nil, nil, fmt.Errorf("source %q: %w", source, err)
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
}
lock, locked := lockSet[source]
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
if err != nil {
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
})
continue
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
skippedOptional = append(skippedOptional, dest)
continue
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
}
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
}
if locked {
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
Source: source,
Dest: dest,
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
Reason: strings.TrimSpace(lock.Reason),
Required: required,
LocalPath: resolved.Path,
Provenance: resolved.Provenance,
})
continue
}
out = append(out, archivePromotion{
Source: source,
@@ -332,7 +395,21 @@ func resolveArchivePromotions(
Provenance: resolved.Provenance,
})
}
return out, skippedOptional, nil
return out, skippedOptional, lockedPromotions, nil
}
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
out := make(map[string]config.ArchiveLockRule, len(locks))
for _, lock := range locks {
source := strings.TrimSpace(lock.Source)
if source == "" {
continue
}
lock.Source = source
lock.Reason = strings.TrimSpace(lock.Reason)
out[source] = lock
}
return out
}
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
@@ -497,6 +574,51 @@ func collectArchiveRunFiles(runRoot, manifestPath string) ([]archiveUploadFile,
return files, nil
}
func collectArchivePreviousFiles(previousDir string) ([]archiveUploadFile, error) {
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
if previousDir == "" {
return nil, fmt.Errorf("previous directory is required")
}
info, err := os.Stat(previousDir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("stat %q: %w", previousDir, err)
}
if !info.IsDir() {
return nil, fmt.Errorf("previous path %q is not a directory", previousDir)
}
files := make([]archiveUploadFile, 0, 16)
err = filepath.WalkDir(previousDir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(previousDir, path)
if err != nil {
return fmt.Errorf("relative path from %q to %q: %w", previousDir, path, err)
}
rel = filepath.ToSlash(rel)
files = append(files, archiveUploadFile{
RelativePath: filepath.ToSlash(filepath.Join(config.PathPreviousDirSegment, rel)),
LocalPath: path,
})
return nil
})
if err != nil {
return nil, fmt.Errorf("walk %q: %w", previousDir, err)
}
sort.Slice(files, func(i, j int) bool {
return files[i].RelativePath < files[j].RelativePath
})
return files, nil
}
func resolveArchiveRunManifestSource(runRoot string) (string, error) {
path := filepath.Join(filepath.Clean(runRoot), "manifest.json")
info, err := os.Stat(path)
@@ -600,7 +722,9 @@ func archiveMetadataPreview(
bucket, runPrefix, sessionPrefix string,
runUploaded []string,
promotedUploaded []string,
previousUploaded []string,
skippedOptional []string,
lockedPromotions []archiveLockedPromotion,
currentManifestKey string,
) map[string]any {
return map[string]any{
@@ -612,10 +736,30 @@ func archiveMetadataPreview(
"run_uploaded_paths": append([]string(nil), runUploaded...),
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": append([]string(nil), promotedUploaded...),
"previous_files_uploaded": len(previousUploaded),
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
"locked_promotion_count": len(lockedPromotions),
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
"current_manifest_key": currentManifestKey,
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
"current_pointer_written": false,
"audio_upload_skipped": true,
}
}
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
out := make([]map[string]any, 0, len(locked))
for _, item := range locked {
out = append(out, map[string]any{
"source": item.Source,
"dest": item.Dest,
"remote_key": item.RemoteKey,
"reason": item.Reason,
"required": item.Required,
"local_path": item.LocalPath,
"provenance": item.Provenance,
})
}
return out
}

View File

@@ -2,6 +2,7 @@ package stage
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -130,6 +131,50 @@ func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
if result.Metadata["promoted_files_uploaded"] != 2 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
}
if result.Metadata["previous_files_uploaded"] != 0 {
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
}
}
func TestArchiveUploadsPreviousCacheWhenPresent(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
sessionRoot := artifacts.SessionWorkDirForCampaign(
env.Config.Pipeline.Workspace.Root,
env.Config.Session.Campaign,
env.Config.Session.SessionID,
)
writeStageTestFile(t, filepath.Join(sessionRoot, "previous", "manifest.json"), "{\"session_id\":\"2026-04-12\"}\n")
writeStageTestFile(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
previousManifestKey := m.S3SessionPrefix + "previous/manifest.json"
previousRecapKey := m.S3SessionPrefix + "previous/artifacts/session_recap.md"
if _, ok := fake.Objects[previousManifestKey]; !ok {
t.Fatalf("missing archived previous manifest key %q", previousManifestKey)
}
if _, ok := fake.Objects[previousRecapKey]; !ok {
t.Fatalf("missing archived previous artifact key %q", previousRecapKey)
}
if result.Metadata["previous_files_uploaded"] != 2 {
t.Fatalf("metadata previous_files_uploaded = %#v, want 2", result.Metadata["previous_files_uploaded"])
}
}
func TestArchiveToleratesMissingPreviousCache(t *testing.T) {
env, m, _ := archiveFixture(t)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["previous_files_uploaded"] != 0 {
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
}
}
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
@@ -171,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
}
}
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
if _, ok := fake.Objects[trimmedKey]; ok {
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
}
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
if _, ok := fake.Objects[recapKey]; !ok {
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
}
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
if _, ok := fake.Objects[runTrimmedKey]; !ok {
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
}
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
}
if result.Metadata["promoted_files_uploaded"] != 1 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
}
if result.Metadata["locked_promotion_count"] != 1 {
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["source"] != "narratio.transcript.trimmed" ||
locked[0]["dest"] != "transcripts/trimmed.json" ||
locked[0]["remote_key"] != trimmedKey ||
locked[0]["reason"] != "human reviewed" ||
locked[0]["required"] != true ||
locked[0]["local_path"] == "" ||
locked[0]["provenance"] == "" {
t.Fatalf("locked promotion metadata = %#v", locked[0])
}
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
var current map[string]any
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, &current); err != nil {
t.Fatalf("unmarshal current manifest: %v", err)
}
stages := current["stages"].(map[string]any)
archive := stages["archive"].(map[string]any)
meta := archive["metadata"].(map[string]any)
if meta["locked_promotion_count"] != float64(1) {
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
}
items := meta["locked_promotions"].([]any)
if len(items) != 1 {
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
}
}
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
}
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
}
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
if _, ok := fake.Objects[mergedKey]; ok {
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
t.Fatalf("current run pointer should be written for locked missing promotion")
}
locked := result.Metadata["locked_promotions"].([]map[string]any)
if len(locked) != 1 {
t.Fatalf("locked_promotions = %#v, want one item", locked)
}
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
}
}
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
{Source: "narratio.transcript.trimmed", Reason: "already published"},
}
fake := env.ObjectStore.(*storage.FakeBackend)
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
got := string(fake.Objects[trimmedKey].Data)
if got != "previously published\n" {
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
}
for _, upload := range fake.Uploads {
if upload.Key == trimmedKey {
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
}
}
}
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{

View File

@@ -30,8 +30,10 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
store := artifacts.NewLocalStore(root)
cfgDir := t.TempDir()
sessionPath := filepath.Join(cfgDir, "session.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
writeStageTestFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
@@ -48,7 +50,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
env := &Env{
Config: &config.Config{
SessionPath: sessionPath,
CampaignPath: campaignPath,
PipelinePath: pipelinePath,
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{
@@ -62,14 +66,28 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
UploadRun: boolPtr(true),
},
},
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
AutocorrectFile: "./autocorrect.yml",
GlossaryFile: "./glossary.yml",
AudioDir: "./audio",
},
},
},

View File

@@ -13,6 +13,7 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/audio"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gopkg.in/yaml.v3"
@@ -25,6 +26,7 @@ func (prepareStage) Name() string { return "prepare" }
func (prepareStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "config", Category: "inputs", RelativePath: "campaign.yml"},
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
@@ -59,21 +61,30 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
}
campaignSrc := env.Config.CampaignPath
if err := requireFile(campaignSrc, "campaign.yml"); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
sessionSrc := env.Config.SessionPath
if err := requireFile(sessionSrc, "session.yml"); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
sessionDir := filepath.Dir(sessionSrc)
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile)
speakersInput := stableInputSource(env.Config.StableInputs.SpeakersFile, env.Config.Session.Inputs.SpeakersFile, sessionSrc)
autocorrectInput := stableInputSource(env.Config.StableInputs.AutocorrectFile, env.Config.Session.Inputs.AutocorrectFile, sessionSrc)
glossaryInput := stableInputSource(env.Config.StableInputs.GlossaryFile, env.Config.Session.Inputs.GlossaryFile, sessionSrc)
speakersSrc, err := resolveConfigRelativePath(speakersInput)
if err != nil {
return nil, fmt.Errorf("prepare: speakers path: %w", err)
}
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile)
autocorrectSrc, err := resolveConfigRelativePath(autocorrectInput)
if err != nil {
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
}
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile)
glossarySrc, err := resolveConfigRelativePath(glossaryInput)
if err != nil {
return nil, fmt.Errorf("prepare: glossary path: %w", err)
}
@@ -96,17 +107,44 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
}
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
inputs := make([]manifest.InputRecord, 0, 6+len(resolvedLocalAudio))
registerInput := func(kind, path, checksum string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
}
registerConfigInput := func(kind, path, checksum, source string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum, Source: source})
}
registerSessionConfigInput := func(path, checksum string) {
source := env.Config.SessionSource
if strings.TrimSpace(source.Source) == "" {
source.Source = "session_config"
}
inputs = append(inputs, manifest.InputRecord{
Kind: "session_config",
Path: path,
Checksum: checksum,
Source: source.Source,
S3Bucket: source.S3Bucket,
S3Key: source.S3Key,
S3Size: source.S3Size,
S3ETag: source.S3ETag,
SpoolPath: source.SpoolPath,
})
}
campaignDst := filepath.Join(paths.InputsDir, "campaign.yml")
campaignChecksum, err := copyFileIfChanged(env.ArtifactStore, campaignSrc, campaignDst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize campaign.yml: %w", err)
}
registerConfigInput("campaign_config", campaignDst, campaignChecksum, "campaign_config")
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
}
registerInput("session_config", sessionDst, sessionChecksum)
registerSessionConfigInput(sessionDst, sessionChecksum)
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
if err != nil {
@@ -120,31 +158,49 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
registerInput("pipeline_resolved", pipelineDst, pipelineChecksum)
for _, cfgFile := range []struct {
kind string
src string
dst string
kind string
src string
dst string
source string
}{
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")},
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")},
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")},
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml"), source: speakersInput.Source},
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml"), source: autocorrectInput.Source},
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml"), source: glossaryInput.Source},
} {
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
}
registerInput(cfgFile.kind, cfgFile.dst, checksum)
registerConfigInput(cfgFile.kind, cfgFile.dst, checksum, cfgFile.source)
}
var audioCacheStats s3AudioMaterializationStats
if useS3Audio {
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
stats, err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs)
if err != nil {
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
}
audioCacheStats = stats
} else {
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
}
previousRequirements := collectPreparePreviousRequirements(env.Config)
var previousHydration *previousSessionHydrationResult
if len(previousRequirements) > 0 {
if err := clearManagedPreviousState(paths); err != nil {
return nil, fmt.Errorf("prepare: clear previous-session cache: %w", err)
}
hydration, err := hydratePreviousSessionArtifacts(ctx, env, paths, previousRequirements)
if err != nil {
return nil, fmt.Errorf("prepare: hydrate previous-session artifacts: %w", err)
}
previousHydration = hydration
inputs = append(inputs, hydration.Inputs...)
}
sort.Slice(inputs, func(i, j int) bool {
if inputs[i].Kind != inputs[j].Kind {
return inputs[i].Kind < inputs[j].Kind
@@ -153,13 +209,32 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
})
m.Inputs = inputs
metadata := map[string]any{
"prepared": true,
"stage": "prepare",
"inputs_count": len(inputs),
"audio_files_resolved": countAudioInputs(inputs),
}
if useS3Audio {
metadata["audio_cache_hits"] = audioCacheStats.CacheHits
metadata["audio_cache_misses"] = audioCacheStats.CacheMisses
metadata["audio_s3_downloads"] = audioCacheStats.Downloads
}
if len(previousRequirements) > 0 {
metadata["previous_requirements_count"] = len(previousRequirements)
if previousHydration != nil {
metadata["previous_artifacts_hydrated"] = append([]string(nil), previousHydration.Hydrated...)
metadata["previous_artifacts_hydrated_count"] = len(previousHydration.Hydrated)
metadata["previous_artifacts_missing_optional"] = append([]string(nil), previousHydration.SkippedMissing...)
metadata["previous_artifacts_missing_optional_count"] = len(previousHydration.SkippedMissing)
if strings.TrimSpace(previousHydration.PreviousRunID) != "" {
metadata["previous_session_run_id"] = strings.TrimSpace(previousHydration.PreviousRunID)
}
}
}
return &StageResult{
Metadata: map[string]any{
"prepared": true,
"stage": "prepare",
"inputs_count": len(inputs),
"audio_files_resolved": countAudioInputs(inputs),
},
Metadata: metadata,
}, nil
}
@@ -167,6 +242,31 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
return yaml.Marshal(cfg)
}
func stableInputSource(resolved config.ResolvedInputFile, fallbackPath, fallbackConfigPath string) config.ResolvedInputFile {
if strings.TrimSpace(resolved.Path) != "" || strings.TrimSpace(resolved.ConfigPath) != "" || strings.TrimSpace(resolved.Source) != "" {
if strings.TrimSpace(resolved.ConfigPath) == "" {
resolved.ConfigPath = fallbackConfigPath
}
if strings.TrimSpace(resolved.Source) == "" {
resolved.Source = "session_config"
}
return resolved
}
return config.ResolvedInputFile{
Path: fallbackPath,
ConfigPath: fallbackConfigPath,
Source: "session_config",
}
}
func resolveConfigRelativePath(input config.ResolvedInputFile) (string, error) {
basePath := strings.TrimSpace(input.ConfigPath)
if basePath == "" {
return "", fmt.Errorf("source config path is required")
}
return resolvePath(filepath.Dir(basePath), input.Path)
}
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
if inputs.AudioS3 != nil {
@@ -254,28 +354,34 @@ func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolve
return nil
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
type s3AudioMaterializationStats struct {
CacheHits int
CacheMisses int
Downloads int
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) (s3AudioMaterializationStats, error) {
if env.ObjectStore == nil {
return fmt.Errorf("s3 audio input requires object store backend")
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires object store backend")
}
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
return s3AudioMaterializationStats{}, fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return fmt.Errorf("session campaign is required for s3 audio input")
return s3AudioMaterializationStats{}, fmt.Errorf("session campaign is required for s3 audio input")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return fmt.Errorf("run id is required for s3 audio input")
return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
objects, err := env.ObjectStore.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
@@ -293,7 +399,7 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
return audioObjects[i].Key < audioObjects[j].Key
})
if len(audioObjects) == 0 {
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
}
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
@@ -303,45 +409,60 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
return s3AudioMaterializationStats{}, fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
}
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
}
seenBase := map[string]string{}
stats := s3AudioMaterializationStats{}
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
for _, obj := range audioObjects {
base := path.Base(obj.Key)
if prev, exists := seenBase[base]; exists && prev != obj.Key {
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
}
seenBase[base] = obj.Key
spoolPath := filepath.Join(spoolAudioDir, base)
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
}
workPath := filepath.Join(workAudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
Store: env.ObjectStore,
Object: obj,
Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
CacheRoot: strings.TrimSpace(env.Config.Pipeline.Cache.Root),
CacheEnabled: cacheEnabled,
SpoolPath: spoolPath,
DestPath: workPath,
})
if err != nil {
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
return s3AudioMaterializationStats{}, err
}
if result.CacheHit {
stats.CacheHits++
} else {
stats.CacheMisses++
}
if result.Downloaded {
stats.Downloads++
}
*inputs = append(*inputs, manifest.InputRecord{
Kind: "audio",
Path: workPath,
Checksum: checksum,
Checksum: result.Checksum,
Source: "s3",
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
S3Key: obj.Key,
S3Size: obj.Size,
S3ETag: obj.ETag,
SpoolPath: spoolPath,
SpoolPath: result.SpoolPath,
CachePath: result.CachePath,
})
}
return nil
return stats, nil
}
func countAudioInputs(inputs []manifest.InputRecord) int {
@@ -354,6 +475,35 @@ func countAudioInputs(inputs []manifest.InputRecord) int {
return count
}
func collectPreparePreviousRequirements(cfg *config.Config) []artifacts.PreviousArtifactRequirement {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
return nil
}
return artifacts.CollectPreviousArtifactRequirements(cfg.Pipeline.Scriptorium.Artifacts)
}
func clearManagedPreviousState(paths artifacts.SessionPaths) error {
previousDir := filepath.Clean(paths.PreviousDir)
sessionRoot := filepath.Clean(paths.Root)
if strings.TrimSpace(previousDir) == "" || strings.TrimSpace(sessionRoot) == "" {
return fmt.Errorf("previous/session root paths are required")
}
if previousDir == sessionRoot {
return fmt.Errorf("refusing to clear session root as previous cache: %q", previousDir)
}
prefix := sessionRoot + string(filepath.Separator)
if !strings.HasPrefix(previousDir, prefix) {
return fmt.Errorf("refusing to clear path outside session root: %q", previousDir)
}
if filepath.Base(previousDir) != config.PathPreviousDirSegment {
return fmt.Errorf("refusing to clear non-previous path %q", previousDir)
}
if err := os.RemoveAll(previousDir); err != nil {
return err
}
return os.MkdirAll(previousDir, 0o755)
}
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return ""

View File

@@ -0,0 +1,475 @@
package stage
import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
const (
preparePreviousInputKindManifest = "previous_manifest"
preparePreviousInputKindArtifact = "previous_artifact"
preparePreviousInputSource = "previous_session_archive.current"
)
// previousSessionHydrationResult captures prepare-time previous-session cache materialization.
type previousSessionHydrationResult struct {
Inputs []manifest.InputRecord
Hydrated []string
SkippedMissing []string
PreviousRunID string
}
func hydratePreviousSessionArtifacts(
ctx context.Context,
env *Env,
paths artifacts.SessionPaths,
requirements []artifacts.PreviousArtifactRequirement,
) (*previousSessionHydrationResult, error) {
if len(requirements) == 0 {
return &previousSessionHydrationResult{}, nil
}
if env == nil || env.Config == nil || env.Config.Session == nil || env.Config.Pipeline == nil {
return nil, fmt.Errorf("resolved config with session/pipeline is required")
}
if env.ArtifactStore == nil {
return nil, fmt.Errorf("artifact store is required")
}
orderedRequirements := append([]artifacts.PreviousArtifactRequirement(nil), requirements...)
sort.Slice(orderedRequirements, func(i, j int) bool {
return orderedRequirements[i].Name < orderedRequirements[j].Name
})
requiredNames := requiredPreviousArtifactNames(orderedRequirements)
optionalNames := optionalPreviousArtifactNames(orderedRequirements)
previousSessionID := strings.TrimSpace(env.Config.Session.PreviousSessionID)
if previousSessionID == "" {
if len(requiredNames) > 0 {
return nil, fmt.Errorf(
"previous_session_id is required for required previous-session artifacts: %s",
strings.Join(requiredNames, ", "),
)
}
return &previousSessionHydrationResult{SkippedMissing: optionalNames}, nil
}
if env.ObjectStore == nil {
return nil, fmt.Errorf("previous-session artifact hydration requires object store backend")
}
if env.Config.Pipeline.Storage.S3 == nil {
return nil, fmt.Errorf("pipeline.storage.s3 configuration is required for previous-session artifact hydration")
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return nil, fmt.Errorf("session campaign is required for previous-session artifact hydration")
}
bucket := strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
if bucket == "" {
return nil, fmt.Errorf("pipeline.storage.s3.bucket is required for previous-session artifact hydration")
}
previousSessionPrefix := artifacts.S3SessionPrefix(
env.Config.Pipeline.Storage.S3.RootPrefix,
campaign,
previousSessionID,
)
currentManifestKey, currentRunIDKey := artifacts.ResolveArchiveCurrentStateKeys(previousSessionPrefix)
result := &previousSessionHydrationResult{}
runPointerExists, err := env.ObjectStore.Exists(ctx, currentRunIDKey)
if err != nil {
return nil, fmt.Errorf("check previous-session current run pointer %q: %w", currentRunIDKey, err)
}
if !runPointerExists {
if len(requiredNames) > 0 {
return nil, fmt.Errorf("required previous-session artifacts unavailable: remote current run pointer missing: %q", currentRunIDKey)
}
result.SkippedMissing = optionalNames
return result, nil
}
runIDTemp, err := downloadObjectToTempStage(ctx, env.ObjectStore, currentRunIDKey, "narratio-prepare-previous-run-id-*.txt")
if err != nil {
return nil, fmt.Errorf("download previous-session current run pointer %q: %w", currentRunIDKey, err)
}
defer func() { _ = os.Remove(runIDTemp) }()
runIDBytes, err := os.ReadFile(runIDTemp)
if err != nil {
return nil, fmt.Errorf("read previous-session current run pointer %q: %w", currentRunIDKey, err)
}
previousRunID := strings.TrimSpace(string(runIDBytes))
if previousRunID == "" {
return nil, fmt.Errorf("previous-session current run pointer %q is empty", currentRunIDKey)
}
result.PreviousRunID = previousRunID
manifestExists, err := env.ObjectStore.Exists(ctx, currentManifestKey)
if err != nil {
return nil, fmt.Errorf("check previous-session current manifest %q: %w", currentManifestKey, err)
}
if !manifestExists {
if len(requiredNames) > 0 {
return nil, fmt.Errorf("required previous-session artifacts unavailable: remote current manifest missing: %q", currentManifestKey)
}
result.SkippedMissing = optionalNames
return result, nil
}
if err := os.MkdirAll(filepath.Dir(paths.PreviousManifestPath), 0o755); err != nil {
return nil, fmt.Errorf("create previous manifest directory: %w", err)
}
if err := env.ObjectStore.Download(ctx, currentManifestKey, paths.PreviousManifestPath); err != nil {
return nil, fmt.Errorf("download previous-session current manifest %q: %w", currentManifestKey, err)
}
manifestStore := &manifest.LocalStore{}
previousManifest, err := manifestStore.Load(ctx, paths.PreviousManifestPath)
if err != nil {
return nil, fmt.Errorf("decode downloaded previous-session manifest %q: %w", currentManifestKey, err)
}
if strings.TrimSpace(previousManifest.SessionID) != previousSessionID {
return nil, fmt.Errorf(
"previous-session manifest session_id %q does not match configured previous_session_id %q",
strings.TrimSpace(previousManifest.SessionID),
previousSessionID,
)
}
if strings.TrimSpace(previousManifest.Campaign) != campaign {
return nil, fmt.Errorf(
"previous-session manifest campaign %q does not match current campaign %q",
strings.TrimSpace(previousManifest.Campaign),
campaign,
)
}
if strings.TrimSpace(previousManifest.RunID) == "" {
return nil, fmt.Errorf("previous-session manifest run_id is required")
}
if strings.TrimSpace(previousManifest.RunID) != previousRunID {
return nil, fmt.Errorf(
"previous-session current run pointer %q references run %q but current manifest run_id is %q",
currentRunIDKey,
previousRunID,
strings.TrimSpace(previousManifest.RunID),
)
}
manifestChecksum, err := env.ArtifactStore.Checksum(paths.PreviousManifestPath)
if err != nil {
return nil, fmt.Errorf("checksum downloaded previous-session manifest: %w", err)
}
result.Inputs = append(result.Inputs, manifest.InputRecord{
Kind: preparePreviousInputKindManifest,
Path: paths.PreviousManifestPath,
Checksum: manifestChecksum,
Source: preparePreviousInputSource,
S3Bucket: bucket,
S3Key: currentManifestKey,
})
for _, requirement := range orderedRequirements {
candidates := previousArtifactRelativePathCandidates(requirement.Name, previousManifest, env.Config)
if len(candidates) == 0 {
if requirement.Required {
return nil, fmt.Errorf(
"required previous-session artifact %q is unavailable in previous-session manifest/archive",
requirement.Name,
)
}
result.SkippedMissing = append(result.SkippedMissing, requirement.Name)
continue
}
selectedRel := ""
selectedKey := ""
for _, candidate := range candidates {
remoteKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, candidate)
exists, err := env.ObjectStore.Exists(ctx, remoteKey)
if err != nil {
return nil, fmt.Errorf("check previous-session artifact object %q: %w", remoteKey, err)
}
if !exists {
continue
}
selectedRel = candidate
selectedKey = remoteKey
break
}
if selectedRel == "" {
if requirement.Required {
return nil, fmt.Errorf(
"required previous-session artifact %q object missing from archive candidate keys",
requirement.Name,
)
}
result.SkippedMissing = append(result.SkippedMissing, requirement.Name)
continue
}
localPath := artifacts.SessionPreviousArtifactPath(paths, selectedRel)
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return nil, fmt.Errorf("create previous-session artifact directory for %q: %w", localPath, err)
}
if err := env.ObjectStore.Download(ctx, selectedKey, localPath); err != nil {
return nil, fmt.Errorf("download previous-session artifact %q from %q: %w", requirement.Name, selectedKey, err)
}
if err := requireNonEmptyFile(localPath, "previous-session artifact "+requirement.Name); err != nil {
return nil, err
}
checksum, err := env.ArtifactStore.Checksum(localPath)
if err != nil {
return nil, fmt.Errorf("checksum previous-session artifact %q: %w", requirement.Name, err)
}
result.Inputs = append(result.Inputs, manifest.InputRecord{
Kind: preparePreviousInputKindArtifact,
Path: localPath,
Checksum: checksum,
Source: preparePreviousInputSource,
S3Bucket: bucket,
S3Key: selectedKey,
})
result.Hydrated = append(result.Hydrated, requirement.Name)
}
sort.Strings(result.Hydrated)
sort.Strings(result.SkippedMissing)
sort.Slice(result.Inputs, func(i, j int) bool {
if result.Inputs[i].Kind != result.Inputs[j].Kind {
return result.Inputs[i].Kind < result.Inputs[j].Kind
}
return result.Inputs[i].Path < result.Inputs[j].Path
})
return result, nil
}
func requiredPreviousArtifactNames(requirements []artifacts.PreviousArtifactRequirement) []string {
names := make([]string, 0, len(requirements))
for _, requirement := range requirements {
if requirement.Required {
names = append(names, strings.TrimSpace(requirement.Name))
}
}
sort.Strings(names)
return names
}
func optionalPreviousArtifactNames(requirements []artifacts.PreviousArtifactRequirement) []string {
names := make([]string, 0, len(requirements))
for _, requirement := range requirements {
if requirement.Required {
continue
}
names = append(names, strings.TrimSpace(requirement.Name))
}
sort.Strings(names)
return names
}
func previousArtifactRelativePathCandidates(
artifactName string,
previousManifest *manifest.Manifest,
cfg *config.Config,
) []string {
candidates := []string{}
appendCandidate := func(v string) {
normalized, err := normalizeArchiveRelativePath(v)
if err != nil {
return
}
candidates = append(candidates, normalized)
}
sourceID := artifacts.ConfiguredArtifactSourceID(artifactName)
if rel, ok := previousManifestArtifactRelativePathBySourceID(previousManifest, sourceID); ok {
appendCandidate(rel)
base := path.Base(rel)
for _, promoted := range previousManifestPromotedPaths(previousManifest) {
if path.Base(promoted) == base {
appendCandidate(promoted)
}
}
}
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Scriptorium != nil {
if artifactCfg, ok := cfg.Pipeline.Scriptorium.Artifacts[artifactName]; ok {
appendCandidate(artifactCfg.OutputPath)
}
}
return dedupeOrderedStrings(candidates)
}
func previousManifestArtifactRelativePathBySourceID(previousManifest *manifest.Manifest, sourceID string) (string, bool) {
if previousManifest == nil || len(previousManifest.Stages) == 0 {
return "", false
}
sourceID = strings.TrimSpace(sourceID)
if sourceID == "" {
return "", false
}
stageNames := make([]string, 0, len(previousManifest.Stages))
if _, ok := previousManifest.Stages["analyze"]; ok {
stageNames = append(stageNames, "analyze")
}
for stageName := range previousManifest.Stages {
if stageName == "analyze" {
continue
}
stageNames = append(stageNames, stageName)
}
start := 0
if len(stageNames) > 0 && stageNames[0] == "analyze" {
start = 1
}
sort.Strings(stageNames[start:])
for _, stageName := range stageNames {
sr := previousManifest.Stages[stageName]
if sr == nil {
continue
}
for _, out := range sr.Outputs {
if strings.TrimSpace(out.SourceID) != sourceID {
continue
}
rel, ok := derivePreviousManifestRelativePath(previousManifest, out.LocalPath)
if ok {
return rel, true
}
}
}
return "", false
}
func derivePreviousManifestRelativePath(previousManifest *manifest.Manifest, localPath string) (string, bool) {
trimmed := strings.TrimSpace(localPath)
if trimmed == "" {
return "", false
}
if !filepath.IsAbs(trimmed) {
normalized, err := normalizeArchiveRelativePath(filepath.ToSlash(trimmed))
if err != nil {
return "", false
}
return normalized, true
}
sessionRoot, ok := previousManifestSessionRoot(previousManifest)
if !ok {
return "", false
}
rel, err := filepath.Rel(sessionRoot, trimmed)
if err != nil {
return "", false
}
normalized, err := normalizeArchiveRelativePath(filepath.ToSlash(rel))
if err != nil {
return "", false
}
return normalized, true
}
func previousManifestSessionRoot(previousManifest *manifest.Manifest) (string, bool) {
if previousManifest == nil {
return "", false
}
runRoot := filepath.Clean(strings.TrimSpace(previousManifest.LocalWorkDir))
runID := strings.TrimSpace(previousManifest.RunID)
if runRoot == "" || runID == "" {
return "", false
}
if filepath.Base(runRoot) != runID {
return "", false
}
runsDir := filepath.Dir(runRoot)
if filepath.Base(runsDir) != config.PathRunsDirSegment {
return "", false
}
return filepath.Dir(runsDir), true
}
func previousManifestPromotedPaths(previousManifest *manifest.Manifest) []string {
if previousManifest == nil || len(previousManifest.Stages) == 0 {
return nil
}
sr := previousManifest.Stages["archive"]
if sr == nil || sr.Metadata == nil {
return nil
}
raw, ok := sr.Metadata["promoted_paths"]
if !ok {
return nil
}
values, ok := raw.([]any)
if !ok {
return nil
}
out := make([]string, 0, len(values))
for _, value := range values {
asString, ok := value.(string)
if !ok {
continue
}
normalized, err := normalizeArchiveRelativePath(asString)
if err != nil {
continue
}
out = append(out, normalized)
}
return dedupeOrderedStrings(out)
}
func dedupeOrderedStrings(values []string) []string {
if len(values) == 0 {
return nil
}
seen := map[string]struct{}{}
out := make([]string, 0, len(values))
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
continue
}
if _, ok := seen[trimmed]; ok {
continue
}
seen[trimmed] = struct{}{}
out = append(out, trimmed)
}
return out
}
func downloadObjectToTempStage(
ctx context.Context,
store interface {
Download(context.Context, string, string) error
},
key, pattern string,
) (string, error) {
tmp, err := os.CreateTemp("", pattern)
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if err := tmp.Close(); err != nil {
_ = os.Remove(path)
return "", fmt.Errorf("close temp file: %w", err)
}
if err := store.Download(ctx, key, path); err != nil {
_ = os.Remove(path)
return "", err
}
return path, nil
}

View File

@@ -0,0 +1,419 @@
package stage
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestHydratePreviousSessionArtifactsDownloadsManifestAndRequiredArtifact(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeArtifactObject: true,
artifactBody: "# previous recap\n",
})
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err != nil {
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
}
if result == nil {
t.Fatal("result is nil")
}
if len(result.Inputs) != 2 {
t.Fatalf("inputs len = %d, want 2", len(result.Inputs))
}
if !containsString(result.Hydrated, "session_recap") {
t.Fatalf("hydrated = %#v, want session_recap", result.Hydrated)
}
if len(result.SkippedMissing) != 0 {
t.Fatalf("skipped missing = %#v, want none", result.SkippedMissing)
}
if _, err := os.Stat(sessionPaths.PreviousManifestPath); err != nil {
t.Fatalf("previous manifest missing: %v", err)
}
recapPath := artifacts.SessionPreviousArtifactPath(sessionPaths, "artifacts/session_recap.md")
if _, err := os.Stat(recapPath); err != nil {
t.Fatalf("previous artifact missing: %v", err)
}
nestedRecapPath := filepath.Join(sessionPaths.PreviousArtifactsDir, "artifacts", "session_recap.md")
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
}
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
if manifestInput == nil {
t.Fatalf("missing input kind %q", preparePreviousInputKindManifest)
}
if manifestInput.Source != preparePreviousInputSource {
t.Fatalf("manifest input source = %q, want %q", manifestInput.Source, preparePreviousInputSource)
}
artifactInput := findInputByKind(result.Inputs, preparePreviousInputKindArtifact)
if artifactInput == nil {
t.Fatalf("missing input kind %q", preparePreviousInputKindArtifact)
}
if artifactInput.Source != preparePreviousInputSource {
t.Fatalf("artifact input source = %q, want %q", artifactInput.Source, preparePreviousInputSource)
}
if artifactInput.Path != recapPath {
t.Fatalf("artifact input path = %q, want %q", artifactInput.Path, recapPath)
}
}
func TestHydratePreviousSessionArtifactsSkipsMissingOptionalArtifact(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: false},
}
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeManifestObject: true,
includeArtifactObject: false,
})
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err != nil {
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
}
if result == nil {
t.Fatal("result is nil")
}
if !containsString(result.SkippedMissing, "session_recap") {
t.Fatalf("skipped missing = %#v, want session_recap", result.SkippedMissing)
}
if len(result.Hydrated) != 0 {
t.Fatalf("hydrated = %#v, want none", result.Hydrated)
}
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
if manifestInput == nil {
t.Fatalf("missing input kind %q", preparePreviousInputKindManifest)
}
if findInputByKind(result.Inputs, preparePreviousInputKindArtifact) != nil {
t.Fatalf("unexpected %q input for missing optional artifact", preparePreviousInputKindArtifact)
}
}
func TestHydratePreviousSessionArtifactsFailsMissingRequiredArtifact(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeArtifactObject: false,
})
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err == nil || !strings.Contains(err.Error(), "required previous-session artifact") {
t.Fatalf("error = %v, want required artifact failure", err)
}
}
func TestHydratePreviousSessionArtifactsFailsRequiredWhenPreviousSessionIDUnset(t *testing.T) {
env, sessionPaths, _ := previousHydrationFixture(t)
env.Config.Session.PreviousSessionID = ""
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err == nil || !strings.Contains(err.Error(), "previous_session_id is required") {
t.Fatalf("error = %v, want previous_session_id required failure", err)
}
}
func TestHydratePreviousSessionArtifactsOptionalWithNoPreviousSessionID(t *testing.T) {
env, sessionPaths, _ := previousHydrationFixture(t)
env.Config.Session.PreviousSessionID = ""
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: false},
}
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err != nil {
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
}
if result == nil {
t.Fatal("result is nil")
}
if len(result.Inputs) != 0 {
t.Fatalf("inputs len = %d, want 0", len(result.Inputs))
}
if !containsString(result.SkippedMissing, "session_recap") {
t.Fatalf("skipped missing = %#v, want session_recap", result.SkippedMissing)
}
if _, err := os.Stat(sessionPaths.PreviousManifestPath); !os.IsNotExist(err) {
t.Fatalf("previous manifest should not be created, stat err = %v", err)
}
}
func TestHydratePreviousSessionArtifactsRespectsCurrentCommitMarker(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: false,
includeArtifactObject: true,
artifactBody: "# previous recap\n",
})
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err == nil || !strings.Contains(err.Error(), "remote current run pointer missing") {
t.Fatalf("error = %v, want current run pointer missing failure", err)
}
}
func TestHydratePreviousSessionArtifactsDoesNotUseLocalPreviousWorkspaceState(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeManifestObject: true,
includeArtifactObject: false,
})
// Write a local previous-session workspace file that should be ignored.
writeFile(t, filepath.Join(seed.PreviousSessionRoot, "artifacts", "session_recap.md"), "# local stale recap\n")
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err == nil || !strings.Contains(err.Error(), "object missing from archive") {
t.Fatalf("error = %v, want remote-object-missing failure", err)
}
}
func TestHydratePreviousSessionArtifactsUsesExplicitStorageKeys(t *testing.T) {
env, sessionPaths, fake := previousHydrationFixture(t)
requirements := []artifacts.PreviousArtifactRequirement{
{Name: "session_recap", Required: true},
}
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeArtifactObject: true,
artifactBody: "# previous recap\n",
})
capture := &preparePreviousCaptureStore{delegate: fake}
env.ObjectStore = capture
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
if err != nil {
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
}
if !containsString(capture.existsKeys, seed.RunPointerKey) {
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.RunPointerKey)
}
if !containsString(capture.existsKeys, seed.ManifestKey) {
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.ManifestKey)
}
if !containsString(capture.existsKeys, seed.ArtifactKey) {
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.ArtifactKey)
}
if !containsString(capture.downloadKeys, seed.RunPointerKey) {
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.RunPointerKey)
}
if !containsString(capture.downloadKeys, seed.ManifestKey) {
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.ManifestKey)
}
if !containsString(capture.downloadKeys, seed.ArtifactKey) {
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.ArtifactKey)
}
}
type previousStateSeedResult struct {
PreviousSessionPrefix string
PreviousSessionRoot string
ManifestKey string
RunPointerKey string
ArtifactKey string
}
type previousStateSeedOptions struct {
includeRunPointerObject bool
includeManifestObject bool
includeArtifactObject bool
artifactBody string
}
func seedPreviousCurrentState(
t *testing.T,
env *Env,
fake *storage.FakeBackend,
options previousStateSeedOptions,
) previousStateSeedResult {
t.Helper()
if !options.includeRunPointerObject && !options.includeManifestObject && !options.includeArtifactObject {
// Keep default behavior deterministic when caller omits explicit flags.
options.includeRunPointerObject = true
options.includeManifestObject = true
}
if options.includeManifestObject == false && options.includeArtifactObject {
options.includeManifestObject = true
}
previousSessionID := strings.TrimSpace(env.Config.Session.PreviousSessionID)
campaign := strings.TrimSpace(env.Config.Session.Campaign)
rootPrefix := strings.TrimSpace(env.Config.Pipeline.Storage.S3.RootPrefix)
previousSessionPrefix := artifacts.S3SessionPrefix(rootPrefix, campaign, previousSessionID)
manifestKey, runPointerKey := artifacts.ResolveArchiveCurrentStateKeys(previousSessionPrefix)
previousRunID := "20260510T010203Z-a1b2c3d4"
previousSessionRoot := filepath.Join(t.TempDir(), "work", campaign, previousSessionID)
artifactLocalPath := filepath.Join(previousSessionRoot, "artifacts", "session_recap.md")
previousManifest := buildPreviousManifestForSeed(
t,
previousSessionID,
campaign,
previousRunID,
filepath.Join(previousSessionRoot, "runs", previousRunID),
artifactLocalPath,
)
if options.includeRunPointerObject || (!options.includeRunPointerObject && !options.includeManifestObject && !options.includeArtifactObject) {
fake.SeedObject(storage.FakeObject{
Key: runPointerKey,
Data: []byte(previousRunID + "\n"),
})
}
if options.includeManifestObject || options.includeArtifactObject {
fake.SeedObject(storage.FakeObject{
Key: manifestKey,
Data: previousManifest,
})
}
artifactKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, "artifacts/session_recap.md")
if options.includeArtifactObject {
body := options.artifactBody
if body == "" {
body = "# previous recap\n"
}
fake.SeedObject(storage.FakeObject{
Key: artifactKey,
Data: []byte(body),
})
}
return previousStateSeedResult{
PreviousSessionPrefix: previousSessionPrefix,
PreviousSessionRoot: previousSessionRoot,
ManifestKey: manifestKey,
RunPointerKey: runPointerKey,
ArtifactKey: artifactKey,
}
}
func previousHydrationFixture(t *testing.T) (*Env, artifacts.SessionPaths, *storage.FakeBackend) {
t.Helper()
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.PreviousSessionID = "2026-05-10"
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
env.Config.Pipeline.Scriptorium = &config.ScriptoriumConfig{
Artifacts: map[string]config.ScriptoriumArtifactConfig{
"session_recap": {
Enabled: true,
OutputPath: "artifacts/session_recap.md",
},
},
}
fake := &storage.FakeBackend{}
env.ObjectStore = fake
paths, err := ensureLayoutForEnv(env, m.SessionID)
if err != nil {
t.Fatalf("ensureLayoutForEnv() error = %v", err)
}
return env, paths, fake
}
func buildPreviousManifestForSeed(
t *testing.T,
sessionID, campaign, runID, runRoot, artifactPath string,
) []byte {
t.Helper()
now := time.Date(2026, 5, 19, 22, 0, 0, 0, time.UTC)
m := manifest.New(sessionID, now)
m.Campaign = campaign
m.RunID = runID
m.LocalWorkDir = runRoot
m.MarkStageSucceeded("analyze", now, []manifest.ArtifactRecord{
{
Kind: "scriptorium_artifact",
SourceID: artifacts.ConfiguredArtifactSourceID("session_recap"),
LocalPath: artifactPath,
},
})
m.MarkStageSucceeded("archive", now, nil)
m.Stages["archive"].Metadata = map[string]any{
"promoted_paths": []string{"artifacts/session_recap.md"},
}
data, err := json.MarshalIndent(m, "", " ")
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
return append(data, '\n')
}
type preparePreviousCaptureStore struct {
delegate storage.ObjectStore
existsKeys []string
downloadKeys []string
}
func (s *preparePreviousCaptureStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
return s.delegate.List(ctx, prefix)
}
func (s *preparePreviousCaptureStore) Download(ctx context.Context, key, localPath string) error {
s.downloadKeys = append(s.downloadKeys, key)
return s.delegate.Download(ctx, key, localPath)
}
func (s *preparePreviousCaptureStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *preparePreviousCaptureStore) Exists(ctx context.Context, key string) (bool, error) {
s.existsKeys = append(s.existsKeys, key)
return s.delegate.Exists(ctx, key)
}
func findInputByKind(inputs []manifest.InputRecord, kind string) *manifest.InputRecord {
for i := range inputs {
if inputs[i].Kind == kind {
return &inputs[i]
}
}
return nil
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}

View File

@@ -35,6 +35,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
paths := sessionPathsForEnv(env, m.SessionID)
for _, p := range []string{
filepath.Join(paths.InputsDir, "campaign.yml"),
filepath.Join(paths.InputsDir, "session.yml"),
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
filepath.Join(paths.InputsDir, "speakers.yml"),
@@ -48,8 +49,8 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
}
}
if len(m.Inputs) != 7 {
t.Fatalf("manifest inputs len = %d, want 7", len(m.Inputs))
if len(m.Inputs) != 8 {
t.Fatalf("manifest inputs len = %d, want 8", len(m.Inputs))
}
for _, in := range m.Inputs {
if in.Checksum == "" {
@@ -151,6 +152,64 @@ func TestPrepareStageIdempotent(t *testing.T) {
}
}
func TestPrepareStageRecordsLocalSessionProvenance(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
sessionInput := findManifestInput(t, m.Inputs, "session_config")
if sessionInput.Source != "session_config" {
t.Fatalf("session source = %q, want session_config", sessionInput.Source)
}
if sessionInput.S3Bucket != "" || sessionInput.S3Key != "" || sessionInput.SpoolPath != "" {
t.Fatalf("local session input has unexpected remote provenance: %#v", sessionInput)
}
}
func TestPrepareStageRecordsRemoteSessionProvenance(t *testing.T) {
env, m := setupPrepareEnv(t)
remoteSessionPath := filepath.Join(t.TempDir(), "downloaded-session.yml")
writeFile(t, remoteSessionPath, "session_id: 2026-05-03\ninputs:\n audio_s3:\n prefix: audio/\n")
env.Config.SessionPath = remoteSessionPath
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.SessionSource = config.SessionSource{
Source: "session_config.s3",
LocalPath: remoteSessionPath,
S3Bucket: "my-dnd-archive",
S3Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml",
S3Size: 58,
S3ETag: "session-etag",
SpoolPath: remoteSessionPath,
}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "sample-campaign", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "sample-campaign", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
env.ObjectStore = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
sessionInput := findManifestInput(t, m.Inputs, "session_config")
if sessionInput.Source != "session_config.s3" {
t.Fatalf("session source = %q, want session_config.s3", sessionInput.Source)
}
if sessionInput.S3Bucket != "my-dnd-archive" || sessionInput.S3Key != "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml" {
t.Fatalf("remote session input missing bucket/key: %#v", sessionInput)
}
if sessionInput.S3Size != 58 || sessionInput.S3ETag != "session-etag" || sessionInput.SpoolPath != remoteSessionPath {
t.Fatalf("remote session input missing metadata: %#v", sessionInput)
}
}
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
@@ -186,6 +245,15 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
t.Fatalf("expected file %q: %v", p, err)
}
}
if got := result.Metadata["audio_cache_hits"]; got != 0 {
t.Fatalf("audio_cache_hits = %#v, want 0", got)
}
if got := result.Metadata["audio_cache_misses"]; got != 2 {
t.Fatalf("audio_cache_misses = %#v, want 2", got)
}
if got := result.Metadata["audio_s3_downloads"]; got != 2 {
t.Fatalf("audio_s3_downloads = %#v, want 2", got)
}
audioInputs := 0
for _, in := range m.Inputs {
@@ -199,15 +267,75 @@ func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
if in.S3Bucket != "my-dnd-archive" {
t.Fatalf("audio input bucket = %q", in.S3Bucket)
}
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
if in.S3Key == "" || in.SpoolPath == "" || in.CachePath == "" || in.Checksum == "" {
t.Fatalf("audio input missing provenance: %#v", in)
}
if _, err := os.Stat(in.CachePath); err != nil {
t.Fatalf("expected cache path %q: %v", in.CachePath, err)
}
}
if audioInputs != 2 {
t.Fatalf("audio input count = %d, want 2", audioInputs)
}
}
func TestPrepareStageS3AudioUsesCacheOnRerun(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
env.ObjectStore = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("first prepare.Run() error = %v", err)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want 1", len(fake.Downloads))
}
if err := os.RemoveAll(filepath.Join(m.LocalWorkDir, "audio")); err != nil {
t.Fatalf("remove work audio: %v", err)
}
if err := os.RemoveAll(m.LocalSpoolDir); err != nil {
t.Fatalf("remove spool audio: %v", err)
}
fake.DownloadErr = os.ErrPermission
m2 := manifest.New("2026-05-03", time.Now().UTC())
m2.RunID = "20260515T041522Z-a1b2c3d4"
m2.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m2.SessionID, m2.RunID)
m2.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m2.SessionID, m2.RunID)
result, err := (prepareStage{}).Run(context.Background(), env, m2)
if err != nil {
t.Fatalf("cached prepare.Run() error = %v", err)
}
if len(fake.Downloads) != 1 {
t.Fatalf("downloads = %d, want cached rerun to avoid new download", len(fake.Downloads))
}
if got := result.Metadata["audio_cache_hits"]; got != 1 {
t.Fatalf("audio_cache_hits = %#v, want 1", got)
}
if got := result.Metadata["audio_s3_downloads"]; got != 0 {
t.Fatalf("audio_s3_downloads = %#v, want 0", got)
}
audioInput := findManifestInput(t, m2.Inputs, "audio")
if audioInput.CachePath == "" {
t.Fatalf("audio input missing cache path: %#v", audioInput)
}
if audioInput.SpoolPath != "" {
t.Fatalf("audio input spool path = %q, want empty on cache hit", audioInput.SpoolPath)
}
mustReadFileEquals(t, filepath.Join(m2.LocalWorkDir, "audio", "alice.flac"), "alice")
}
func TestPrepareStageS3AudioFailures(t *testing.T) {
tests := []struct {
name string
@@ -284,6 +412,192 @@ func TestPrepareStageAudioSourceConflictFails(t *testing.T) {
}
}
func TestPrepareStageWithoutPreviousRequirementsDoesNotTouchPreviousState(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
paths := sessionPathsForEnv(env, m.SessionID)
stalePath := filepath.Join(paths.PreviousDir, "stale.txt")
writeFile(t, stalePath, "stale")
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := os.Stat(stalePath); err != nil {
t.Fatalf("expected previous stale file to remain untouched: %v", err)
}
}
func TestPrepareStageOptionalPreviousArtifactWithoutPreviousSessionIDSucceeds(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
configurePreparePreviousArtifactSource(env, false)
env.Config.Session.PreviousSessionID = ""
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.PreviousArtifactsDir, "stale.txt"), "stale")
result, err := (prepareStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if result.Metadata["previous_requirements_count"] != 1 {
t.Fatalf("metadata previous_requirements_count = %#v, want 1", result.Metadata["previous_requirements_count"])
}
if result.Metadata["previous_artifacts_hydrated_count"] != 0 {
t.Fatalf("metadata previous_artifacts_hydrated_count = %#v, want 0", result.Metadata["previous_artifacts_hydrated_count"])
}
if result.Metadata["previous_artifacts_missing_optional_count"] != 1 {
t.Fatalf("metadata previous_artifacts_missing_optional_count = %#v, want 1", result.Metadata["previous_artifacts_missing_optional_count"])
}
if _, err := os.Stat(filepath.Join(paths.PreviousArtifactsDir, "stale.txt")); !os.IsNotExist(err) {
t.Fatalf("expected stale previous state to be cleared, stat err = %v", err)
}
for _, in := range m.Inputs {
if in.Kind == preparePreviousInputKindManifest || in.Kind == preparePreviousInputKindArtifact {
t.Fatalf("unexpected previous input record: %#v", in)
}
}
}
func TestPrepareStageRequiredPreviousArtifactWithoutPreviousSessionIDFails(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
configurePreparePreviousArtifactSource(env, true)
env.Config.Session.PreviousSessionID = ""
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "previous_session_id is required") {
t.Fatalf("error = %v, want previous_session_id required failure", err)
}
}
func TestPrepareStageHydratesRequiredPreviousArtifactAndRecordsInputs(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
env.Config.Session.Campaign = "forsaken"
env.Config.Session.PreviousSessionID = "2026-05-10"
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
configurePreparePreviousArtifactSource(env, true)
fake := &storage.FakeBackend{}
env.ObjectStore = fake
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeArtifactObject: true,
artifactBody: "# prior recap\n",
})
paths := sessionPathsForEnv(env, m.SessionID)
result, err := (prepareStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := os.Stat(paths.PreviousManifestPath); err != nil {
t.Fatalf("expected previous manifest: %v", err)
}
recapPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
if _, err := os.Stat(recapPath); err != nil {
t.Fatalf("expected previous artifact: %v", err)
}
nestedRecapPath := filepath.Join(paths.PreviousArtifactsDir, "artifacts", "session_recap.md")
if _, err := os.Stat(nestedRecapPath); !os.IsNotExist(err) {
t.Fatalf("nested previous artifact should not be created, stat err = %v", err)
}
var hasPreviousManifest, hasPreviousArtifact bool
for _, in := range m.Inputs {
if in.Kind == preparePreviousInputKindManifest {
hasPreviousManifest = true
}
if in.Kind == preparePreviousInputKindArtifact {
hasPreviousArtifact = true
}
}
if !hasPreviousManifest || !hasPreviousArtifact {
t.Fatalf("manifest inputs missing previous provenance: %#v", m.Inputs)
}
if result.Metadata["previous_artifacts_hydrated_count"] != 1 {
t.Fatalf("metadata previous_artifacts_hydrated_count = %#v, want 1", result.Metadata["previous_artifacts_hydrated_count"])
}
}
func TestPrepareStageRerunOverwritesPreviousCache(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
env.Config.Session.Campaign = "forsaken"
env.Config.Session.PreviousSessionID = "2026-05-10"
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
configurePreparePreviousArtifactSource(env, true)
fake := &storage.FakeBackend{}
env.ObjectStore = fake
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
includeRunPointerObject: true,
includeArtifactObject: true,
artifactBody: "# old recap\n",
})
paths := sessionPathsForEnv(env, m.SessionID)
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("first prepare run error = %v", err)
}
recapPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
firstBytes, err := os.ReadFile(recapPath)
if err != nil {
t.Fatalf("read first hydrated artifact: %v", err)
}
if strings.TrimSpace(string(firstBytes)) != "# old recap" {
t.Fatalf("first hydrated content = %q, want %q", strings.TrimSpace(string(firstBytes)), "# old recap")
}
writeFile(t, filepath.Join(paths.PreviousArtifactsDir, "stale.txt"), "stale")
fake.SeedObject(storage.FakeObject{Key: seed.ArtifactKey, Data: []byte("# new recap\n")})
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("second prepare run error = %v", err)
}
secondBytes, err := os.ReadFile(recapPath)
if err != nil {
t.Fatalf("read second hydrated artifact: %v", err)
}
if strings.TrimSpace(string(secondBytes)) != "# new recap" {
t.Fatalf("second hydrated content = %q, want %q", strings.TrimSpace(string(secondBytes)), "# new recap")
}
if _, err := os.Stat(filepath.Join(paths.PreviousArtifactsDir, "stale.txt")); !os.IsNotExist(err) {
t.Fatalf("expected stale previous cache file to be removed, stat err = %v", err)
}
}
func configurePreparePreviousArtifactSource(env *Env, required bool) {
env.Config.Pipeline.Scriptorium = &config.ScriptoriumConfig{
Artifacts: map[string]config.ScriptoriumArtifactConfig{
"session_recap": {
Enabled: true,
OutputPath: "artifacts/session_recap.md",
Inputs: map[string]config.ScriptoriumInputConfig{
"previous_recap": {
Source: "narratio.previous_session.artifact.session_recap",
Required: required,
},
},
},
},
}
}
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
t.Helper()
workspace := t.TempDir()
@@ -291,25 +605,51 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
writeFile(t, campaignPath, `campaign: sample-campaign
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`)
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
cfg := &config.Config{
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: workspace},
Cache: config.CacheConfig{Root: filepath.Join(t.TempDir(), "cache"), S3Audio: boolPtr(true)},
},
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
SessionPath: sessionPath,
CampaignPath: campaignPath,
PipelinePath: pipelinePath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
AutocorrectFile: "./autocorrect.yml",
GlossaryFile: "./glossary.yml",
AudioDir: "./audio",
},
},
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{
Path: "./speakers.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
AutocorrectFile: config.ResolvedInputFile{
Path: "./autocorrect.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
GlossaryFile: config.ResolvedInputFile{
Path: "./glossary.yml",
ConfigPath: campaignPath,
Source: "campaign_config",
},
},
}
@@ -337,3 +677,25 @@ func snapshotInputs(inputs []manifest.InputRecord) map[string]string {
}
return out
}
func findManifestInput(t *testing.T, inputs []manifest.InputRecord, kind string) manifest.InputRecord {
t.Helper()
for _, input := range inputs {
if input.Kind == kind {
return input
}
}
t.Fatalf("manifest input kind %q not found in %#v", kind, inputs)
return manifest.InputRecord{}
}
func mustReadFileEquals(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
if string(data) != want {
t.Fatalf("%q = %q, want %q", path, string(data), want)
}
}

View File

@@ -7,6 +7,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -99,6 +100,9 @@ func runLocalPathForCanonical(layout runStageLayout, sessionPaths artifacts.Sess
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("canonical path %q is outside session root %q", cleanCanonical, sessionPaths.Root)
}
if rel == config.PathPreviousDirSegment || strings.HasPrefix(rel, config.PathPreviousDirSegment+string(filepath.Separator)) {
return cleanCanonical, nil
}
localPath := filepath.Join(layout.OutputsDir, rel)
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return "", fmt.Errorf("create run-local output parent for %q: %w", localPath, err)

View File

@@ -33,3 +33,24 @@ func TestRunLocalPathForCanonicalCreatesParentDirectories(t *testing.T) {
t.Fatalf("expected run-local parent directory to exist: %v", err)
}
}
func TestRunLocalPathForCanonicalKeepsPreviousStateSessionDurable(t *testing.T) {
root := t.TempDir()
sessionRoot := filepath.Join(root, "work", "dilfs", "2026-05-17")
layout := runStageLayout{
Enabled: true,
OutputsDir: filepath.Join(sessionRoot, "runs", "run-1", "prepare", "outputs"),
}
if err := os.MkdirAll(layout.OutputsDir, 0o755); err != nil {
t.Fatalf("mkdir outputs dir: %v", err)
}
canonical := filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md")
got, err := runLocalPathForCanonical(layout, artifacts.SessionPaths{Root: sessionRoot}, canonical)
if err != nil {
t.Fatalf("runLocalPathForCanonical() error = %v", err)
}
if got != canonical {
t.Fatalf("runLocalPathForCanonical() = %q, want canonical %q", got, canonical)
}
}

View File

@@ -233,8 +233,10 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
campaignPath := filepath.Join(cfgDir, "campaign.yml")
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
writeFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -243,7 +245,9 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
concurrency := 2
cfg := &config.Config{
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: workspace},
WhisperX: config.WhisperXConfig{
@@ -265,6 +269,11 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
GlossaryFile: "./glossary.yml",
},
},
StableInputs: config.ResolvedStableInputs{
SpeakersFile: config.ResolvedInputFile{Path: "./speakers.yml", ConfigPath: campaignPath, Source: "campaign_config"},
AutocorrectFile: config.ResolvedInputFile{Path: "./autocorrect.yml", ConfigPath: campaignPath, Source: "campaign_config"},
GlossaryFile: config.ResolvedInputFile{Path: "./glossary.yml", ConfigPath: campaignPath, Source: "campaign_config"},
},
}
env := &Env{