Compare commits
18 Commits
v0.11.0
...
7111edeca4
| Author | SHA1 | Date | |
|---|---|---|---|
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 | |||
| bc2ade38d9 | |||
| 5be831eb13 | |||
| cae4d99a89 | |||
| e09dc0512d | |||
| ae82bc1ce0 | |||
| 01eb7aa1aa | |||
| 2ca700195c | |||
| 2b08c34539 | |||
| 79f1fc1e09 | |||
| 9c753270bd | |||
| b907cb01aa | |||
| 7824afd4a5 | |||
| 2a4e1e912c |
36
docs/cli.md
36
docs/cli.md
@@ -6,7 +6,7 @@
|
|||||||
narratio run --session-id 2026-04-04
|
narratio run --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
This command uses default discovery for `pipeline.yml` and `session.yml`; both files must be discoverable unless you pass explicit `--config` and `--session` paths.
|
This command uses default discovery for `pipeline.yml`, `campaign.yml`, and local `session.yml`. If local session discovery misses and S3 storage is configured, `--session-id` can load remote `session.yml` from the canonical session prefix.
|
||||||
|
|
||||||
## Command Overview
|
## Command Overview
|
||||||
|
|
||||||
@@ -28,31 +28,39 @@ For config semantics, see [docs/config.md](./config.md). For operator lifecycle
|
|||||||
### `run`
|
### `run`
|
||||||
|
|
||||||
- `--config <path>`: optional explicit `pipeline.yml` path.
|
- `--config <path>`: optional explicit `pipeline.yml` path.
|
||||||
|
- `--campaign <path>`: optional explicit `campaign.yml` path.
|
||||||
- `--session <path>`: optional explicit `session.yml` path.
|
- `--session <path>`: optional explicit `session.yml` path.
|
||||||
- `--session-id <value>`: session template variable value.
|
- `--session-id <value>`: session template variable value.
|
||||||
|
- `--previous-session-id <value>`: previous-session template variable value.
|
||||||
- `--force`: force stage execution.
|
- `--force`: force stage execution.
|
||||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||||
|
|
||||||
### `plan`
|
### `plan`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
- `--force`
|
- `--force`
|
||||||
|
|
||||||
### `resume`
|
### `resume`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
- `--force`
|
- `--force`
|
||||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||||
|
|
||||||
### `run-stage`
|
### `run-stage`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
- `--force`
|
- `--force`
|
||||||
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
- `--artifacts <names>`: analyze artifact keys to execute (repeatable or comma-separated).
|
||||||
- positional `<stage>`: required stage name.
|
- positional `<stage>`: required stage name.
|
||||||
@@ -72,8 +80,10 @@ Valid stage names:
|
|||||||
### `restore`
|
### `restore`
|
||||||
|
|
||||||
- `--config <path>`
|
- `--config <path>`
|
||||||
|
- `--campaign <path>`
|
||||||
- `--session <path>`
|
- `--session <path>`
|
||||||
- `--session-id <value>`
|
- `--session-id <value>`
|
||||||
|
- `--previous-session-id <value>`
|
||||||
- `--dry-run`: plan restore actions without writing local files.
|
- `--dry-run`: plan restore actions without writing local files.
|
||||||
- `--force`: overwrite local conflicting files with remote archive files.
|
- `--force`: overwrite local conflicting files with remote archive files.
|
||||||
- `--include-audio`: include durable archived `audio/**` files in restore scope.
|
- `--include-audio`: include durable archived `audio/**` files in restore scope.
|
||||||
@@ -92,14 +102,15 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio run [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||||
|
|
||||||
Common failure cases:
|
Common failure cases:
|
||||||
- missing default config/session paths when flags omitted.
|
- missing default config/campaign/session paths when flags omitted.
|
||||||
|
- missing local session plus missing/unavailable remote `session.yml`.
|
||||||
- invalid template/rendered session mismatch.
|
- invalid template/rendered session mismatch.
|
||||||
- unknown/invalid `--artifacts` value.
|
- unknown/invalid `--artifacts` value.
|
||||||
- `--artifacts` with unknown configured artifact key.
|
- `--artifacts` with unknown configured artifact key.
|
||||||
@@ -112,7 +123,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio plan [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force]
|
narratio plan [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output includes:
|
Success output includes:
|
||||||
@@ -121,7 +132,8 @@ Success output includes:
|
|||||||
- `totals: run=<n> skip=<n>`
|
- `totals: run=<n> skip=<n>`
|
||||||
|
|
||||||
Common failure cases:
|
Common failure cases:
|
||||||
- same config/session discovery and validation failures as `run`.
|
- same config/campaign/session discovery and validation failures as `run`.
|
||||||
|
- remote session fallback failures when local session discovery misses.
|
||||||
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
|
- secrets directory read failures when `pipeline.secrets.env_dir` is configured.
|
||||||
|
|
||||||
### `resume`
|
### `resume`
|
||||||
@@ -132,7 +144,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio resume [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio resume [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
@@ -172,7 +184,7 @@ Purpose:
|
|||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
narratio run-stage [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>] <stage>
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output:
|
Success output:
|
||||||
@@ -191,12 +203,12 @@ Common failure cases:
|
|||||||
### `restore`
|
### `restore`
|
||||||
|
|
||||||
Purpose:
|
Purpose:
|
||||||
- Restore durable session state (`manifest.json`, `transcripts/**`, `artifacts/**`, and optional `audio/**`) from the committed remote archive current state.
|
- Restore durable session state (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, and optional `audio/**`) from the committed remote archive current state.
|
||||||
|
|
||||||
Syntax:
|
Syntax:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio restore [--config <pipeline.yml>] [--session <session.yml>] [--session-id <id>] [--dry-run] [--force] [--include-audio]
|
narratio restore [--config <pipeline.yml>] [--campaign <campaign.yml>] [--session <session.yml>] [--session-id <id>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
||||||
```
|
```
|
||||||
|
|
||||||
Success output (dry-run):
|
Success output (dry-run):
|
||||||
@@ -260,6 +272,12 @@ narratio restore --session-id 2026-04-04
|
|||||||
narratio run-stage --session-id 2026-04-04 --force analyze
|
narratio run-stage --session-id 2026-04-04 --force analyze
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Rehydrate canonical previous-session inputs after artifact-input changes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage --session-id 2026-04-04 --force prepare
|
||||||
|
```
|
||||||
|
|
||||||
## Diagnostic / Recovery Commands
|
## Diagnostic / Recovery Commands
|
||||||
|
|
||||||
Inspect stage status:
|
Inspect stage status:
|
||||||
|
|||||||
122
docs/config.md
122
docs/config.md
@@ -2,12 +2,13 @@
|
|||||||
|
|
||||||
## 1. Overview
|
## 1. Overview
|
||||||
|
|
||||||
Narratio loads two YAML files:
|
Narratio loads three YAML files:
|
||||||
|
|
||||||
- `pipeline.yml`: pipeline-level runtime configuration.
|
- `pipeline.yml`: pipeline-level runtime configuration.
|
||||||
- `session.yml`: per-session metadata and input selection.
|
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
|
||||||
|
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
|
||||||
|
|
||||||
These commands load and validate both files before running:
|
These commands load and validate all three files before running:
|
||||||
|
|
||||||
- `narratio run`
|
- `narratio run`
|
||||||
- `narratio plan`
|
- `narratio plan`
|
||||||
@@ -19,7 +20,10 @@ Behavior:
|
|||||||
|
|
||||||
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
||||||
- session templates render before session YAML decode.
|
- session templates render before session YAML decode.
|
||||||
|
- remote `session.yml` uses the same strict decode and template behavior as local `session.yml`.
|
||||||
- defaults are applied for optional pipeline fields.
|
- defaults are applied for optional pipeline fields.
|
||||||
|
- campaign-level stable input paths fill missing session input paths.
|
||||||
|
- session-level stable input paths override campaign-level input paths.
|
||||||
- validation enforces required fields, value formats, and cross-field constraints.
|
- validation enforces required fields, value formats, and cross-field constraints.
|
||||||
|
|
||||||
## 2. Config file discovery
|
## 2. Config file discovery
|
||||||
@@ -32,25 +36,41 @@ Pipeline config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
|||||||
2. `/etc/narratio/pipeline.yml`
|
2. `/etc/narratio/pipeline.yml`
|
||||||
- first existing file wins.
|
- first existing file wins.
|
||||||
|
|
||||||
|
Campaign config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
||||||
|
|
||||||
|
- if `--campaign <path>` is provided, that path is used.
|
||||||
|
- if omitted, Narratio searches in order:
|
||||||
|
1. `./campaign.yml`
|
||||||
|
2. `/usr/local/etc/narratio/campaign.yml`
|
||||||
|
3. `/etc/narratio/campaign.yml`
|
||||||
|
- first existing file wins.
|
||||||
|
|
||||||
## 3. Session file discovery and templating
|
## 3. Session file discovery and templating
|
||||||
|
|
||||||
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
Session config lookup for `run`, `plan`, `resume`, `run-stage`, and `restore`:
|
||||||
|
|
||||||
- if `--session <path>` is provided, that path is used.
|
- if `--session <path>` is provided, that path is used.
|
||||||
- if omitted, Narratio searches in order:
|
- if `--session` is omitted, Narratio searches locally in order:
|
||||||
1. `./session.yml`
|
1. `./session.yml`
|
||||||
2. `/usr/local/etc/narratio/session.yml`
|
2. `/usr/local/etc/narratio/session.yml`
|
||||||
3. `/etc/narratio/session.yml`
|
3. `/etc/narratio/session.yml`
|
||||||
- first existing file wins.
|
- first existing local file wins.
|
||||||
|
- if no local session file is found, `--session-id <value>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
|
||||||
|
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||||
|
- local discovery always runs before remote fallback.
|
||||||
|
|
||||||
Template behavior:
|
Template behavior:
|
||||||
|
|
||||||
- supported placeholders:
|
- supported placeholders:
|
||||||
- `{{session_id}}`
|
- `{{session_id}}`
|
||||||
- `{{ session_id }}`
|
- `{{ session_id }}`
|
||||||
|
- `{{previous_session_id}}`
|
||||||
|
- `{{ previous_session_id }}`
|
||||||
- `--session-id <value>` supplies the placeholder value.
|
- `--session-id <value>` supplies the placeholder value.
|
||||||
|
- `--previous-session-id <value>` supplies the previous-session placeholder value.
|
||||||
- unresolved placeholders fail load.
|
- unresolved placeholders fail load.
|
||||||
- if rendered `session_id` mismatches `--session-id`, load fails.
|
- if rendered `session_id` mismatches `--session-id`, load fails.
|
||||||
|
- if rendered `previous_session_id` mismatches `--previous-session-id`, load fails.
|
||||||
|
|
||||||
## 4. Minimal pipeline config
|
## 4. Minimal pipeline config
|
||||||
|
|
||||||
@@ -67,20 +87,41 @@ Why this is sufficient:
|
|||||||
|
|
||||||
## 5. Minimal session template
|
## 5. Minimal session template
|
||||||
|
|
||||||
|
`campaign.yml`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
session_id: "{{ session_id }}"
|
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
`session.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
session_id: "{{ session_id }}"
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-05-03
|
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03
|
||||||
|
```
|
||||||
|
|
||||||
|
Previous-session-enabled variant:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
session_id: "{{ session_id }}"
|
||||||
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --config /path/to/pipeline.yml --campaign ./campaign.yml --session ./session.yml --session-id 2026-05-03 --previous-session-id 2026-04-26
|
||||||
```
|
```
|
||||||
|
|
||||||
## 6. Production-oriented config
|
## 6. Production-oriented config
|
||||||
@@ -113,6 +154,9 @@ archive:
|
|||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
dest: artifacts/session_recap.md
|
dest: artifacts/session_recap.md
|
||||||
required: true
|
required: true
|
||||||
|
locks:
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
reason: Final recap was manually edited.
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
@@ -127,14 +171,18 @@ scriptorium:
|
|||||||
transcript:
|
transcript:
|
||||||
source: narratio.transcript.trimmed
|
source: narratio.transcript.trimmed
|
||||||
required: true
|
required: true
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session_recap
|
||||||
|
required: false
|
||||||
```
|
```
|
||||||
|
|
||||||
Operational notes:
|
Operational notes:
|
||||||
|
|
||||||
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
||||||
- `source` is required; `dest` is optional and derived when omitted.
|
- `source` is required; `dest` is optional and derived when omitted.
|
||||||
|
- `archive.locks` skips top-level promotion overwrites for locked sources while preserving run-local uploads.
|
||||||
- Narratio does not auto-promote all generated analyze artifacts.
|
- Narratio does not auto-promote all generated analyze artifacts.
|
||||||
- `restore` reads the same config/session inputs and restore scope is bounded by committed archive current state.
|
- `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
|
||||||
|
|
||||||
## 7. Full pipeline reference
|
## 7. Full pipeline reference
|
||||||
|
|
||||||
@@ -161,6 +209,9 @@ Operational notes:
|
|||||||
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
||||||
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
||||||
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
||||||
|
| `pipeline.archive.locks[]` | list | No | empty |
|
||||||
|
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
|
||||||
|
| `pipeline.archive.locks[].reason` | string | No | empty |
|
||||||
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
||||||
| `pipeline.whisperx.language` | string | No | `en` |
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
||||||
@@ -243,13 +294,14 @@ Scriptorium artifact-key and dependency rules:
|
|||||||
|
|
||||||
Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
||||||
|
|
||||||
- `previous_session_artifact`
|
- `narratio.previous_session.artifact.<configured_artifact_key>`
|
||||||
- `narratio.transcript.merged`
|
- `narratio.transcript.merged`
|
||||||
- `narratio.transcript.polished`
|
- `narratio.transcript.polished`
|
||||||
- `narratio.transcript.full`
|
- `narratio.transcript.full`
|
||||||
- `narratio.transcript.trimmed`
|
- `narratio.transcript.trimmed`
|
||||||
- `narratio.bounds.session`
|
- `narratio.bounds.session`
|
||||||
- `narratio.artifact.<configured_artifact_key>`
|
- `narratio.artifact.<configured_artifact_key>`
|
||||||
|
- `previous_session_artifact` (legacy path-based source; uses `inputs.<key>.path`)
|
||||||
|
|
||||||
`pipeline.archive.promote_artifacts[].source` values:
|
`pipeline.archive.promote_artifacts[].source` values:
|
||||||
|
|
||||||
@@ -260,6 +312,8 @@ Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
|||||||
- `narratio.bounds.session`
|
- `narratio.bounds.session`
|
||||||
- `narratio.artifact.<configured_artifact_key>`
|
- `narratio.artifact.<configured_artifact_key>`
|
||||||
|
|
||||||
|
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
|
||||||
|
|
||||||
Archive promotion destination rules:
|
Archive promotion destination rules:
|
||||||
|
|
||||||
- `dest` must be a clean relative path (not absolute, no traversal).
|
- `dest` must be a clean relative path (not absolute, no traversal).
|
||||||
@@ -269,25 +323,47 @@ Archive promotion destination rules:
|
|||||||
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
||||||
- derivation failure is a config validation error.
|
- derivation failure is a config validation error.
|
||||||
|
|
||||||
|
Archive lock rules:
|
||||||
|
|
||||||
|
- locks are source-based and do not accept `dest`.
|
||||||
|
- duplicate lock sources are rejected.
|
||||||
|
- locked promotions are recorded as intentional skips in archive metadata.
|
||||||
|
- locked required promotions do not fail archive by default.
|
||||||
|
- ordinary `--force` reruns do not override locks.
|
||||||
|
|
||||||
Restore-related implications:
|
Restore-related implications:
|
||||||
|
|
||||||
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
||||||
- restore scope considers only committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, optional `audio/**`).
|
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
|
||||||
|
|
||||||
## 8. Full session reference
|
## 8. Full campaign reference
|
||||||
|
|
||||||
|
| Path | Type | Required | Default |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `campaign.campaign` | string | Yes | none |
|
||||||
|
| `campaign.inputs.speakers_file` | string | Yes | none |
|
||||||
|
| `campaign.inputs.autocorrect_file` | string | Yes | none |
|
||||||
|
| `campaign.inputs.glossary_file` | string | Yes | none |
|
||||||
|
|
||||||
|
Campaign input paths may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
|
||||||
|
|
||||||
|
## 9. Full session reference
|
||||||
|
|
||||||
| Path | Type | Required | Default |
|
| Path | Type | Required | Default |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `session.session_id` | string | Yes | none |
|
| `session.session_id` | string | Yes | none |
|
||||||
| `session.campaign` | string | Yes | none |
|
| `session.previous_session_id` | string | No | empty |
|
||||||
|
| `session.campaign` | string | No | `campaign.campaign` |
|
||||||
| `session.date` | string | No | empty |
|
| `session.date` | string | No | empty |
|
||||||
| `session.title` | string | No | empty |
|
| `session.title` | string | No | empty |
|
||||||
| `session.inputs.audio_dir` | string | Conditional | empty |
|
| `session.inputs.audio_dir` | string | Conditional | empty |
|
||||||
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
||||||
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
||||||
| `session.inputs.speakers_file` | string | Yes | none |
|
| `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
|
||||||
| `session.inputs.autocorrect_file` | string | Yes | none |
|
| `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
|
||||||
| `session.inputs.glossary_file` | string | Yes | none |
|
| `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
|
||||||
|
|
||||||
|
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
|
||||||
|
|
||||||
Audio-source rule:
|
Audio-source rule:
|
||||||
|
|
||||||
@@ -297,7 +373,12 @@ Audio-source rule:
|
|||||||
- `audio_s3.prefix`
|
- `audio_s3.prefix`
|
||||||
- `audio_s3` cannot be combined with local audio fields.
|
- `audio_s3` cannot be combined with local audio fields.
|
||||||
|
|
||||||
## 9. Secrets
|
Previous-session rule:
|
||||||
|
|
||||||
|
- if `session.previous_session_id` is set, it must not equal `session.session_id`.
|
||||||
|
- canonical previous-session sources (`narratio.previous_session.artifact.<name>`) are hydrated during `prepare` from archive current state when required by enabled configured artifacts.
|
||||||
|
|
||||||
|
## 10. Secrets
|
||||||
|
|
||||||
Narratio supports filesystem-based secret injection via `pipeline.secrets.env_dir`.
|
Narratio supports filesystem-based secret injection via `pipeline.secrets.env_dir`.
|
||||||
|
|
||||||
@@ -316,13 +397,14 @@ Guidance:
|
|||||||
- do not put secret values directly in YAML.
|
- do not put secret values directly in YAML.
|
||||||
- configure env var names in config and provide values via env/secrets files.
|
- configure env var names in config and provide values via env/secrets files.
|
||||||
|
|
||||||
## 10. Examples
|
## 11. Examples
|
||||||
|
|
||||||
Maintained examples:
|
Maintained examples:
|
||||||
|
|
||||||
- `examples/pipeline.minimal.yml`
|
- `examples/pipeline.minimal.yml`
|
||||||
- `examples/pipeline.production.yml`
|
- `examples/pipeline.production.yml`
|
||||||
- `examples/pipeline.full.annotated.yml`
|
- `examples/pipeline.full.annotated.yml`
|
||||||
|
- `examples/campaign.yml`
|
||||||
- `examples/session.template.yml`
|
- `examples/session.template.yml`
|
||||||
- `examples/session.local-audio.yml`
|
- `examples/session.local-audio.yml`
|
||||||
- `examples/session.s3-audio.yml`
|
- `examples/session.s3-audio.yml`
|
||||||
|
|||||||
@@ -1,41 +1,36 @@
|
|||||||
# Internal: Artifacts
|
# Internal: Artifacts
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Define Narratio's artifact identity and resolution model for built-in transcript/bounds artifacts and runtime-configured analyze artifacts.
|
Define Narratio artifact identity, catalog, and source-resolution behavior for:
|
||||||
|
- built-in session artifacts;
|
||||||
|
- configured analyze artifacts;
|
||||||
|
- canonical previous-session artifact sources.
|
||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- artifact sources from config/runtime (`pipeline.scriptorium.artifacts.*.inputs.*.source`)
|
- configured input sources (`pipeline.scriptorium.artifacts.*.inputs.*.source`);
|
||||||
- session paths and optional session manifest stage outputs
|
- session paths and manifest inputs/outputs;
|
||||||
- runtime artifact catalog state for configured artifact sources
|
- runtime catalog state.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- resolved local artifact path and provenance (`ResolvedSessionArtifact`)
|
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
||||||
- runtime catalog entries for planned/executable/available artifacts
|
- runtime catalog entries for built-ins and configured artifacts;
|
||||||
- validation errors for unsupported, missing, or invalid artifact sources
|
- requirement sets for canonical previous-session inputs.
|
||||||
|
- canonical S3 session, run, current, session config, audio, and promoted artifact keys.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
- built-in artifact registry and content validation rules
|
- built-in source registry and validation;
|
||||||
- runtime artifact catalog for configured artifact source IDs
|
- configured artifact catalog identity (`narratio.artifact.<name>`);
|
||||||
- source resolution behavior for built-in and configured artifact sources
|
- canonical previous-session source parsing and resolution;
|
||||||
|
- previous-session requirement collection (`CollectPreviousArtifactRequirements`).
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- artifact generation (stages produce files)
|
- prepare-stage remote hydration;
|
||||||
- manifest transition policy
|
- stage success/skip transitions;
|
||||||
- archive promotion behavior
|
- archive upload orchestration.
|
||||||
|
|
||||||
## Config fields used
|
|
||||||
- `pipeline.scriptorium.artifacts.<name>.enabled`
|
|
||||||
- `pipeline.scriptorium.artifacts.<name>.output_path`
|
|
||||||
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
|
|
||||||
|
|
||||||
## External adapters used
|
|
||||||
- none
|
|
||||||
|
|
||||||
## State and manifest behavior
|
|
||||||
Built-in registry entries:
|
|
||||||
|
|
||||||
|
## Built-in IDs
|
||||||
| Artifact ID | Canonical file | Producer stage | Output kind |
|
| Artifact ID | Canonical file | Producer stage | Output kind |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` |
|
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` |
|
||||||
@@ -44,44 +39,66 @@ Built-in registry entries:
|
|||||||
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` |
|
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` |
|
||||||
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` |
|
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` |
|
||||||
|
|
||||||
Runtime catalog entries include built-ins and configured `narratio.artifact.<name>` sources.
|
## Source families
|
||||||
|
- built-in: `narratio.transcript.*`, `narratio.bounds.session`
|
||||||
|
- configured artifact: `narratio.artifact.<artifact_key>`
|
||||||
|
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
Catalog states:
|
## S3 key helpers
|
||||||
- `planned`: source is registered and known for this run
|
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||||
- `executable`: configured artifact is selected for analyze execution
|
- session config: `{session_prefix}/session.yml`
|
||||||
- `available`: artifact has a usable file path (generated this run or reused from disk)
|
- run prefix: `{session_prefix}/runs/{run_id}/`
|
||||||
|
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
|
||||||
|
- current manifest: `{session_prefix}/current/manifest.json`
|
||||||
|
- current run pointer: `{session_prefix}/current/run_id.txt`
|
||||||
|
|
||||||
Resolution behavior:
|
## Runtime catalog model
|
||||||
- built-in sources resolve via manifest producer outputs first, then canonical fallback path
|
Catalog entries track:
|
||||||
- configured `narratio.artifact.<name>` sources resolve through runtime catalog availability
|
- `planned`: source is registered for this run;
|
||||||
- configured source lookup requires catalog context
|
- `executable`: configured artifact is selected for analyze execution;
|
||||||
|
- `available`: usable local file exists (generated this run or reused from disk).
|
||||||
|
|
||||||
Configured artifact provenance values:
|
Configured artifact provenance values include:
|
||||||
- `generated.current_analyze_run`
|
- `generated.current_analyze_run`
|
||||||
- `filesystem.disabled_artifact_output`
|
- `filesystem.disabled_artifact_output`
|
||||||
|
|
||||||
Content validation:
|
Previous-session canonical provenance values include:
|
||||||
- transcript built-ins: JSON with top-level `segments` array
|
- `manifest.inputs.previous_cache`
|
||||||
- bounds built-in: valid JSON
|
- `current_session.previous_cache`
|
||||||
- configured artifacts: non-empty text file
|
|
||||||
|
|
||||||
## Skip and resume behavior
|
## Resolution behavior
|
||||||
- resolver and catalog have no direct skip/resume decisions
|
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
||||||
- stage/runner skip-resume behavior consumes catalog/resolver results
|
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
||||||
|
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
||||||
|
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
||||||
|
|
||||||
|
## Previous-session requirement scanning
|
||||||
|
`CollectPreviousArtifactRequirements`:
|
||||||
|
- scans enabled configured artifacts only;
|
||||||
|
- includes canonical previous-session sources only;
|
||||||
|
- deduplicates by artifact key;
|
||||||
|
- merges required/optional references (`required` wins);
|
||||||
|
- records deterministic sorted source locations for diagnostics.
|
||||||
|
|
||||||
|
## Validation behavior
|
||||||
|
- transcript built-ins: JSON with top-level `segments` array;
|
||||||
|
- bounds built-in: valid JSON;
|
||||||
|
- configured and previous-session artifact files: non-empty text content.
|
||||||
|
|
||||||
## Failure behavior
|
## Failure behavior
|
||||||
- unsupported source -> source validation error
|
- unsupported source or malformed canonical previous source: validation/resolution error;
|
||||||
- known source unavailable -> `ErrSessionArtifactNotFound`
|
- known source unavailable: `ErrSessionArtifactNotFound`;
|
||||||
- configured source without catalog -> resolution error
|
- configured/previous canonical source without catalog: error;
|
||||||
- resolved file with invalid content -> validation error
|
- resolved invalid file content: validation error.
|
||||||
|
|
||||||
## Tests to inspect before changing
|
## Tests to inspect before changing
|
||||||
- `internal/artifacts/artifact_resolver_test.go`
|
- `internal/artifacts/artifact_resolver_test.go`
|
||||||
- `internal/artifacts/catalog_test.go`
|
- `internal/artifacts/catalog_test.go`
|
||||||
|
- `internal/artifacts/previous_requirements_test.go`
|
||||||
|
- `internal/stage/prepare_previous_test.go`
|
||||||
- `internal/stage/analyze_test.go`
|
- `internal/stage/analyze_test.go`
|
||||||
- `internal/config/scriptorium_test.go`
|
|
||||||
|
|
||||||
## Architectural invariants
|
## Architectural invariants
|
||||||
- built-in IDs are static and registry-backed
|
- Built-in source IDs are static.
|
||||||
- configured artifact IDs are runtime-derived (`narratio.artifact.<name>`) and catalog-backed
|
- Configured and previous-session source IDs are artifact-key based and validation-gated.
|
||||||
- built-in/source resolution remains deterministic and validation-gated
|
- Resolution behavior remains deterministic and manifest-aware.
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
# Internal: Command Restore
|
# Internal: Command Restore
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Define the implemented `narratio restore` command contract: committed remote-state discovery, deterministic planning, safe file installation, conflict policy, and restore reporting.
|
Define the implemented `narratio restore` contract: committed remote-state discovery, deterministic plan classification, safe file install semantics, and restore reporting.
|
||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- CLI flags: `--config`, `--session`, `--session-id`, `--dry-run`, `--force`, `--include-audio`.
|
- CLI flags: `--config`, `--session`, `--session-id`, `--previous-session-id`, `--dry-run`, `--force`, `--include-audio`.
|
||||||
- Resolved/validated `pipeline.yml` and `session.yml`.
|
- Resolved/validated `pipeline.yml` and `session.yml`.
|
||||||
- Configured remote object store.
|
- Configured remote object store.
|
||||||
- Remote committed current-state markers (`current/run_id.txt`, `current/manifest.json`).
|
- Remote committed current-state markers (`current/run_id.txt`, `current/manifest.json`).
|
||||||
@@ -54,6 +54,21 @@ Does not own:
|
|||||||
- existing local manifest is preserved if restored manifest validation/install fails.
|
- existing local manifest is preserved if restored manifest validation/install fails.
|
||||||
- Non-dry-run report persists summary/action status metadata in `reports/restore-latest.json`.
|
- Non-dry-run report persists summary/action status metadata in `reports/restore-latest.json`.
|
||||||
|
|
||||||
|
Restore path scope:
|
||||||
|
- includes:
|
||||||
|
- `manifest.json`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**`
|
||||||
|
- `audio/**` only when `--include-audio` is set
|
||||||
|
- excludes:
|
||||||
|
- `runs/**`
|
||||||
|
- `logs/**`
|
||||||
|
- `reports/**`
|
||||||
|
- `config/**`
|
||||||
|
- `inputs/**`
|
||||||
|
- remote `current/**` pointer files as local restore targets
|
||||||
|
|
||||||
## Skip and resume behavior
|
## Skip and resume behavior
|
||||||
- Restore does not participate in stage skip/resume decisions.
|
- Restore does not participate in stage skip/resume decisions.
|
||||||
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
|
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
|
||||||
@@ -80,7 +95,7 @@ Does not own:
|
|||||||
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
|
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
|
||||||
- Local path mapping is traversal-safe and constrained to session root.
|
- Local path mapping is traversal-safe and constrained to session root.
|
||||||
- Restore scope is deterministic and path-classified:
|
- Restore scope is deterministic and path-classified:
|
||||||
- include `manifest.json`, `transcripts/**`, `artifacts/**`
|
- include `manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`
|
||||||
- include `audio/**` only with `--include-audio`
|
- include `audio/**` only with `--include-audio`
|
||||||
- exclude `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`
|
- exclude `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`
|
||||||
- Command remains standalone; no implicit `run --restore` behavior.
|
- Command remains standalone; no implicit `run --restore` behavior.
|
||||||
|
|||||||
@@ -3,31 +3,36 @@
|
|||||||
## Purpose
|
## Purpose
|
||||||
Execute selected configured Scriptorium artifacts in deterministic dependency order and promote successful outputs to canonical session artifact paths.
|
Execute selected configured Scriptorium artifacts in deterministic dependency order and promote successful outputs to canonical session artifact paths.
|
||||||
|
|
||||||
## Inputs and Outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- configured artifact definitions from `pipeline.scriptorium.artifacts`
|
- configured artifact definitions from `pipeline.scriptorium.artifacts`;
|
||||||
- selected artifact filter from runtime (`--artifacts`) when provided
|
- selected artifact filter (`--artifacts`) when provided;
|
||||||
- resolved artifact input sources declared per artifact (`inputs.*.source`)
|
- resolved artifact sources from resolver/catalog.
|
||||||
- optional previous-session file inputs (`previous_session_artifact`)
|
|
||||||
|
Source types used by analyze:
|
||||||
|
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`;
|
||||||
|
- configured artifacts: `narratio.artifact.<artifact_key>`;
|
||||||
|
- canonical previous-session artifacts: `narratio.previous_session.artifact.<artifact_key>`;
|
||||||
|
- legacy path-based previous-session source: `previous_session_artifact` (uses `inputs.*.path`).
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- one promoted output file per executed configured artifact at that artifact's configured `output_path`
|
- promoted configured artifact files at each configured `output_path`;
|
||||||
- stage metadata containing generated artifact entries and reused disabled-artifact entries
|
- stage metadata (`generated_artifacts`, `reused_artifacts`, selected/order info).
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
- runtime artifact catalog construction for analyze execution
|
- runtime artifact catalog construction;
|
||||||
- selected-artifact planning and dependency ordering
|
- selected-artifact planning and dependency ordering;
|
||||||
- per-artifact input resolution, var resolution, timeout/render-debug resolution
|
- per-input resolution and required/optional handling;
|
||||||
- Scriptorium run/render invocation for each selected artifact
|
- Scriptorium render/run invocation;
|
||||||
- run-local output generation and canonical promotion
|
- run-local output generation and canonical promotion.
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- transcript generation/processing stages
|
- prepare-time previous-session hydration;
|
||||||
- archive promotion policy
|
- object-store access for previous-session sources;
|
||||||
- per-artifact resume semantics
|
- archive promotion policy.
|
||||||
|
|
||||||
## Config Fields Used
|
## Config fields used
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
@@ -36,49 +41,41 @@ Does not own:
|
|||||||
- `pipeline.scriptorium.timeout`
|
- `pipeline.scriptorium.timeout`
|
||||||
- `pipeline.scriptorium.render_debug`
|
- `pipeline.scriptorium.render_debug`
|
||||||
- `pipeline.scriptorium.artifacts.<name>.*`
|
- `pipeline.scriptorium.artifacts.<name>.*`
|
||||||
- `enabled`
|
|
||||||
- `depends_on`
|
|
||||||
- `prompt_id`
|
|
||||||
- `profile_id`
|
|
||||||
- `timeout`
|
|
||||||
- `output_path`
|
|
||||||
- `render_debug`
|
|
||||||
- `inputs`
|
|
||||||
- `vars`
|
|
||||||
|
|
||||||
## External Adapters Used
|
## External adapters used
|
||||||
- Scriptorium adapter:
|
- Scriptorium adapter:
|
||||||
- optional `RenderArtifact` (render debug)
|
- optional `RenderArtifact` when render-debug is enabled;
|
||||||
- `RunArtifact` (artifact generation)
|
- `RunArtifact` for artifact generation.
|
||||||
|
|
||||||
## State and Manifest Behavior
|
## State and manifest behavior
|
||||||
- If `pipeline.scriptorium` is absent, stage returns success metadata with `skipped=true`.
|
- If Scriptorium config is absent, or no artifacts are executable after filtering, analyze returns success metadata with `skipped=true`.
|
||||||
- If no artifacts are configured, stage returns success metadata with `skipped=true`.
|
- Builds runtime catalog with built-ins and configured `narratio.artifact.<name>` entries.
|
||||||
- If zero artifacts are executable after `enabled` + `--artifacts` filtering, stage returns success metadata with `skipped=true`.
|
- Non-executable configured artifacts may still be marked available from existing canonical output files.
|
||||||
- Builds runtime catalog with built-ins and configured artifacts.
|
- Resolves canonical previous-session sources from local prepared `previous/` cache:
|
||||||
- Non-executable configured artifacts are marked available only when their configured output file exists and is valid on disk.
|
- prefers manifest-backed previous input paths when present;
|
||||||
- Executes selected configured artifacts in topological order with deterministic tie-breaking.
|
- may fall back to current-session `previous/` filesystem paths.
|
||||||
- For each generated artifact, records metadata fields including `name`, `source_id`, `output_kind`, `path`, `prompt_id`, `profile_id`, and `provenance`.
|
- Analyze does not call object storage for canonical previous-session source resolution.
|
||||||
- Reused disabled artifacts are recorded separately in `reused_artifacts` with provenance `filesystem.disabled_artifact_output`.
|
- Required canonical previous-session input missing:
|
||||||
|
- fails with guidance to run `narratio run-stage --force prepare`.
|
||||||
|
- Optional missing sources are omitted from adapter input paths.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and resume behavior
|
||||||
- Runner-level skip applies when analyze is already `succeeded` and `--force` is not set.
|
- Runner-level skip applies when analyze is already `succeeded` and `--force` is not set.
|
||||||
- Analyze remains stage-scoped for resume/skip; there is no per-artifact resume state.
|
- Analyze is stage-scoped for resume; no per-artifact manifest resume state.
|
||||||
- `--artifacts` filters which configured artifacts are executable when analyze runs; it does not imply `--force`.
|
- `--artifacts` filters executable artifacts but does not imply force rerun.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure behavior
|
||||||
- Fails on invalid dependency ordering, unavailable required configured inputs, invalid built-in input prerequisites, render/run adapter failures, validation-failed adapter results, or missing/empty outputs.
|
- Fails on dependency-order violations, missing required inputs, resolver validation failures, adapter errors, and missing/empty generated outputs.
|
||||||
- Required configured dependency missing from catalog availability fails clearly before invocation.
|
- Required unavailable configured artifact source (`narratio.artifact.<name>`) fails before invocation.
|
||||||
- Optional missing inputs are omitted.
|
- Required canonical previous-session source fails with prepare-rerun guidance.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Tests to inspect before changing
|
||||||
- `internal/stage/analyze_test.go`
|
- `internal/stage/analyze_test.go`
|
||||||
- `internal/artifacts/catalog_test.go`
|
- `internal/artifacts/catalog_test.go`
|
||||||
- `internal/artifacts/artifact_resolver_test.go`
|
- `internal/artifacts/artifact_resolver_test.go`
|
||||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
- `internal/app/restore_workflow_test.go`
|
||||||
|
|
||||||
## Architectural Invariants
|
## Architectural invariants
|
||||||
- Configured artifacts are identified by `narratio.artifact.<name>` source IDs.
|
- Canonical previous-session behavior is local-cache only during analyze.
|
||||||
- Artifact-to-artifact references rely on explicit `depends_on` declarations validated in config.
|
- Generated outputs are validated and promoted before stage success is recorded.
|
||||||
- Generated analyze outputs are treated uniformly as Scriptorium artifacts.
|
- Resolver/catalog decisions stay deterministic and validation-gated.
|
||||||
- Successful outputs must exist and be non-empty before promotion.
|
|
||||||
|
|||||||
@@ -1,17 +1,20 @@
|
|||||||
# Stage: archive
|
# Stage: archive
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Publish run records and promoted session artifacts to object storage, then atomically advance the remote current pointer.
|
Publish durable run/session state to object storage, then atomically advance remote current state.
|
||||||
|
|
||||||
## Inputs and Outputs
|
## Inputs and Outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- session manifest and prerequisite stage records
|
- session manifest and prerequisite stage records
|
||||||
- run root contents under `runs/{run_id}/`
|
- run root contents under `runs/{run_id}/`
|
||||||
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
||||||
|
- source-based promotion locks (`archive.locks`)
|
||||||
|
- session-level `previous/**` cache files when present
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
|
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
|
||||||
- uploaded promoted artifacts under `{session_prefix}/...`
|
- uploaded promoted artifacts under `{session_prefix}/...`
|
||||||
|
- uploaded session previous-cache files under `{session_prefix}/previous/...` when present
|
||||||
- `{session_prefix}/current/manifest.json`
|
- `{session_prefix}/current/manifest.json`
|
||||||
- `{session_prefix}/current/run_id.txt` written last
|
- `{session_prefix}/current/run_id.txt` written last
|
||||||
|
|
||||||
@@ -21,6 +24,8 @@ Owns:
|
|||||||
- Prerequisite stage success enforcement
|
- Prerequisite stage success enforcement
|
||||||
- Run file collection and upload (excluding `audio/`)
|
- Run file collection and upload (excluding `audio/`)
|
||||||
- Promotion rule resolution and upload
|
- Promotion rule resolution and upload
|
||||||
|
- Promotion lock enforcement
|
||||||
|
- Session previous-cache file collection/upload
|
||||||
- Commit pointer publish order
|
- Commit pointer publish order
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
@@ -31,6 +36,7 @@ Does not own:
|
|||||||
- `pipeline.archive.enabled`
|
- `pipeline.archive.enabled`
|
||||||
- `pipeline.archive.upload_run`
|
- `pipeline.archive.upload_run`
|
||||||
- `pipeline.archive.promote_artifacts`
|
- `pipeline.archive.promote_artifacts`
|
||||||
|
- `pipeline.archive.locks`
|
||||||
- `pipeline.storage.s3.bucket`
|
- `pipeline.storage.s3.bucket`
|
||||||
- `pipeline.storage.s3.root_prefix`
|
- `pipeline.storage.s3.root_prefix`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
@@ -43,8 +49,12 @@ Does not own:
|
|||||||
## State and Manifest Behavior
|
## State and Manifest Behavior
|
||||||
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
||||||
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
||||||
|
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
||||||
|
- Skips top-level promotion uploads for sources listed in `pipeline.archive.locks`; run-local uploads still publish.
|
||||||
- Writes metadata including:
|
- Writes metadata including:
|
||||||
- upload counts/paths
|
- upload counts/paths
|
||||||
|
- `previous_files_uploaded` and `previous_uploaded_paths`
|
||||||
|
- `locked_promotion_count` and `locked_promotions`
|
||||||
- `current_manifest_key`
|
- `current_manifest_key`
|
||||||
- `current_run_id_key`
|
- `current_run_id_key`
|
||||||
- `current_pointer_written`
|
- `current_pointer_written`
|
||||||
@@ -55,7 +65,8 @@ Does not own:
|
|||||||
- Runner-level skip also applies for previously succeeded stage unless forced.
|
- Runner-level skip also applies for previously succeeded stage unless forced.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
- Fails on missing prerequisite success, missing object store when required, missing run root, missing required promotion source, upload failures, or pointer write failures.
|
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
|
||||||
|
- Locked required promotions are intentional skips and do not fail archive.
|
||||||
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Tests to Inspect Before Changing
|
||||||
@@ -64,5 +75,7 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Run upload excludes `audio/` subtree.
|
- Run upload excludes `audio/` subtree.
|
||||||
|
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
|
||||||
|
- Ordinary `--force` does not override archive locks.
|
||||||
- `current/manifest.json` uploads before `current/run_id.txt`.
|
- `current/manifest.json` uploads before `current/run_id.txt`.
|
||||||
- `current/run_id.txt` is the remote publish commit marker.
|
- `current/run_id.txt` is the remote publish commit marker.
|
||||||
|
|||||||
@@ -1,40 +1,51 @@
|
|||||||
# Stage: prepare
|
# Stage: prepare
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Materialize all required session inputs into canonical local workspace paths and record input provenance in the session manifest.
|
Materialize canonical current-session input state and provenance before downstream stages run.
|
||||||
|
|
||||||
## Inputs and Outputs
|
Prepare owns:
|
||||||
|
- local input file materialization (`inputs/**`);
|
||||||
|
- audio input materialization (`audio/**`);
|
||||||
|
- previous-session cache hydration (`previous/**`) for canonical previous-session artifact sources.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- `session.yml` (resolved session config)
|
- resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
|
||||||
- `pipeline.resolved.yml` (materialized from resolved pipeline config)
|
- remote session provenance when `session.yml` was loaded from S3;
|
||||||
- `speakers.yml`
|
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
|
||||||
- `autocorrect.yml`
|
|
||||||
- `glossary.yml`
|
|
||||||
- audio source:
|
- audio source:
|
||||||
- local (`session.inputs.audio_dir` or `session.inputs.audio_files`), or
|
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
||||||
- S3 (`session.inputs.audio_s3.prefix`)
|
- S3: `session.inputs.audio_s3.prefix`;
|
||||||
|
- configured enabled Scriptorium artifact inputs (for previous-session requirement scanning);
|
||||||
|
- remote previous-session current archive state when previous hydration is required.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- `inputs/session.yml`
|
- `inputs/campaign.yml`;
|
||||||
- `inputs/pipeline.resolved.yml`
|
- `inputs/session.yml`;
|
||||||
- `inputs/speakers.yml`
|
- `inputs/pipeline.resolved.yml`;
|
||||||
- `inputs/autocorrect.yml`
|
- `inputs/speakers.yml`;
|
||||||
- `inputs/glossary.yml`
|
- `inputs/autocorrect.yml`;
|
||||||
- `audio/*.flac` in session workdir
|
- `inputs/glossary.yml`;
|
||||||
- `manifest.Inputs` records with checksums and source metadata
|
- `audio/*.flac` in canonical session `audio/`;
|
||||||
|
- optional `previous/manifest.json`;
|
||||||
|
- optional `previous/artifacts/**`;
|
||||||
|
- deterministic `manifest.Inputs` records with checksums and provenance metadata.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
- Input path resolution and validation
|
- input path resolution and materialization;
|
||||||
- Local copy/materialization of configs and audio files
|
- S3 audio list/download/copy flow;
|
||||||
- S3 audio download to run-scoped spool, then copy into work audio dir
|
- previous-session artifact requirement collection from enabled configured artifacts;
|
||||||
|
- previous cache lifecycle when requirements exist (clear and rehydrate managed `previous/` state).
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Transcript generation/processing
|
- transcript or artifact generation;
|
||||||
- Archive publish behavior
|
- analyze-stage source resolution;
|
||||||
|
- archive commit behavior.
|
||||||
|
|
||||||
## Config Fields Used
|
## Config fields used
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
|
- `session.previous_session_id`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
- `session.inputs.speakers_file`
|
- `session.inputs.speakers_file`
|
||||||
- `session.inputs.autocorrect_file`
|
- `session.inputs.autocorrect_file`
|
||||||
@@ -46,29 +57,63 @@ Does not own:
|
|||||||
- `pipeline.spool.root`
|
- `pipeline.spool.root`
|
||||||
- `pipeline.storage.s3.bucket`
|
- `pipeline.storage.s3.bucket`
|
||||||
- `pipeline.storage.s3.root_prefix`
|
- `pipeline.storage.s3.root_prefix`
|
||||||
|
- `pipeline.scriptorium.artifacts.<name>.enabled`
|
||||||
|
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
|
||||||
|
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
|
||||||
|
- `campaign.campaign`
|
||||||
|
- `campaign.inputs.speakers_file`
|
||||||
|
- `campaign.inputs.autocorrect_file`
|
||||||
|
- `campaign.inputs.glossary_file`
|
||||||
|
|
||||||
## External Adapters Used
|
## External adapters used
|
||||||
- Object storage backend (`env.ObjectStore`) for S3 audio list/download when `audio_s3` is configured.
|
- `storage.ObjectStore` for:
|
||||||
|
- S3 audio listing/downloads;
|
||||||
|
- previous-session current pointer/manifest/artifact object checks and downloads.
|
||||||
|
|
||||||
## State and Manifest Behavior
|
## State and manifest behavior
|
||||||
- Ensures workspace layout exists.
|
- Ensures workspace layout exists.
|
||||||
- Writes resolved config and input files to canonical `inputs/` paths.
|
- Materializes canonical input files and audio files.
|
||||||
- Records all prepared inputs into `manifest.Inputs` (sorted deterministically by kind/path).
|
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
|
||||||
- For S3 audio, records `S3Bucket`, `S3Key`, `S3Size`, `S3ETag`, and `SpoolPath` in each audio input record.
|
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
|
||||||
|
- Resolves session-provided stable input overrides relative to `session.yml`.
|
||||||
|
- Scans enabled configured artifact inputs for canonical sources:
|
||||||
|
- `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
- If one or more canonical previous-session requirements exist:
|
||||||
|
- clears managed `previous/` state;
|
||||||
|
- hydrates required/optional previous artifacts from the configured previous session’s committed archive current state;
|
||||||
|
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
||||||
|
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
|
||||||
|
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
||||||
|
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Required and optional previous-session behavior
|
||||||
- Runner-level skip applies when stage already `succeeded` and `--force` is not set.
|
- `previous_session_id` unset:
|
||||||
- Stage itself is deterministic/idempotent for unchanged inputs (`copyFileIfChanged`, `writeBytesIfChanged`).
|
- if any referenced previous artifact is required: fail;
|
||||||
|
- if all referenced previous artifacts are optional: continue and omit them.
|
||||||
|
- Previous session archive current pointer or manifest missing:
|
||||||
|
- if any referenced previous artifact is required: fail;
|
||||||
|
- if all referenced previous artifacts are optional: continue and omit missing ones.
|
||||||
|
- Missing required previous artifact object: fail.
|
||||||
|
- Missing optional previous artifact object: omit.
|
||||||
|
- Downloaded previous artifacts must validate as non-empty files.
|
||||||
|
|
||||||
## Failure Behavior
|
## Skip and resume behavior
|
||||||
- Fails on missing required files, invalid audio source combinations, no discoverable `.flac` files, duplicate audio basenames, missing object store for S3 mode, or S3 list/download failures.
|
- Runner-level skip remains authoritative:
|
||||||
|
- if `prepare` already succeeded and run is not forced, `prepare` does not run and no hydration/download occurs.
|
||||||
|
- If `prepare` runs (including with `--force`), it owns managed `previous/` state for canonical previous-session inputs.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Failure behavior
|
||||||
|
- Fails on missing required input files, invalid audio-source combinations, empty/duplicate audio inputs, missing object store for S3 modes, and remote access/download/validation errors.
|
||||||
|
- For required canonical previous-session inputs, analyze-time missing-input guidance is to rerun:
|
||||||
|
- `narratio run-stage --force prepare`
|
||||||
|
|
||||||
|
## Tests to inspect before changing
|
||||||
- `internal/stage/prepare_test.go`
|
- `internal/stage/prepare_test.go`
|
||||||
- `internal/app/session_cli_test.go`
|
- `internal/stage/prepare_previous_test.go`
|
||||||
- `internal/config/load_validate_test.go`
|
- `internal/artifacts/previous_requirements_test.go`
|
||||||
|
- `internal/app/runner_test.go`
|
||||||
|
|
||||||
## Architectural Invariants
|
## Architectural invariants
|
||||||
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
|
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
|
||||||
- Audio files must be `.flac`.
|
- Storage keys are computed by callers using archive/path helpers; storage adapter receives explicit keys.
|
||||||
- Canonical `inputs/*` and `audio/*` paths are the durable source for downstream stages.
|
- `prepare` is the only stage that hydrates canonical previous-session cache state.
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ Document Narratio's remote storage backend contracts and implementations under `
|
|||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
- Resolved storage config (`pipeline.storage.*`).
|
- Resolved storage config (`pipeline.storage.*`).
|
||||||
- Bucket-relative object keys and local file paths from stage/app orchestration.
|
- Bucket-relative object keys and local file paths from app/stage orchestration.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
|
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
|
||||||
@@ -57,6 +57,7 @@ Implementations:
|
|||||||
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
|
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
|
||||||
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
|
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
|
||||||
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
|
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
|
||||||
|
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
|
||||||
|
|
||||||
## Tests to inspect before changing
|
## Tests to inspect before changing
|
||||||
- `internal/adapters/storage/factory_test.go`
|
- `internal/adapters/storage/factory_test.go`
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ Outputs:
|
|||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
- Session-level path layout (`inputs/`, `audio/`, `transcripts/`, `artifacts/`, `reports/`, `logs/`, `config/`, `current/`, `runs/`)
|
- Session-level path layout (`inputs/`, `audio/`, `transcripts/`, `artifacts/`, `reports/`, `logs/`, `config/`, `current/`, `runs/`, `previous/`)
|
||||||
|
- `previous/manifest.json` and `previous/artifacts/**` are reserved for prepared previous-session state
|
||||||
- Run-local stage sandbox layout under `runs/{run_id}/{stage}/`
|
- Run-local stage sandbox layout under `runs/{run_id}/{stage}/`
|
||||||
- Session lock acquisition/release (`.lock`)
|
- Session lock acquisition/release (`.lock`)
|
||||||
|
|
||||||
@@ -65,4 +66,5 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
|
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
|
||||||
- Run roots are always nested: `runs/{run_id}` under the session root.
|
- Run roots are always nested: `runs/{run_id}` under the session root.
|
||||||
- Run-local output promotion must end in canonical session paths.
|
- Run-local output promotion must end in canonical session paths.
|
||||||
|
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
||||||
- Cleanup only targets run-scoped directories and must never delete configured root directories.
|
- Cleanup only targets run-scoped directories and must never delete configured root directories.
|
||||||
|
|||||||
@@ -6,19 +6,21 @@ For field-level configuration, see [docs/config.md](./config.md). For full comma
|
|||||||
|
|
||||||
## Normal workflow (S3-first path)
|
## Normal workflow (S3-first path)
|
||||||
|
|
||||||
1. Upload session `.flac` files to object storage under the configured session audio prefix.
|
1. Upload `session.yml` to the configured session prefix, or keep a local `session.yml` available.
|
||||||
2. Run Narratio:
|
2. Upload session `.flac` files to object storage under the configured session audio prefix.
|
||||||
|
3. Run Narratio:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run --session-id 2026-04-04
|
narratio run --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
3. Read success output:
|
4. Read success output:
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
||||||
- use `manifest=<path>` with `status` for inspection.
|
- use `manifest=<path>` with `status` for inspection.
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
- default config/session discovery applies unless `--config` and `--session` are passed.
|
- default config/campaign/session discovery applies unless `--config`, `--campaign`, and `--session` are passed.
|
||||||
|
- when local `session.yml` discovery misses, `--session-id` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
||||||
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
||||||
|
|
||||||
## Restore workflow
|
## Restore workflow
|
||||||
@@ -48,7 +50,7 @@ Restore source-of-truth:
|
|||||||
- remote current manifest: `current/manifest.json`
|
- remote current manifest: `current/manifest.json`
|
||||||
|
|
||||||
Restore default scope:
|
Restore default scope:
|
||||||
- includes `manifest.json`, `transcripts/**`, `artifacts/**`
|
- includes `manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`
|
||||||
- includes `audio/**` only with `--include-audio`
|
- includes `audio/**` only with `--include-audio`
|
||||||
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
||||||
|
|
||||||
@@ -61,12 +63,14 @@ Primary state:
|
|||||||
- `manifest.json`: session-level stage state.
|
- `manifest.json`: session-level stage state.
|
||||||
- `runs/{run_id}/manifest.json`: invocation-level state.
|
- `runs/{run_id}/manifest.json`: invocation-level state.
|
||||||
- `.lock`: session lock while a modifying command is active.
|
- `.lock`: session lock while a modifying command is active.
|
||||||
|
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
|
||||||
|
|
||||||
Canonical session directories:
|
Canonical session directories:
|
||||||
- `inputs/`
|
- `inputs/`
|
||||||
- `audio/`
|
- `audio/`
|
||||||
- `transcripts/`
|
- `transcripts/`
|
||||||
- `artifacts/`
|
- `artifacts/`
|
||||||
|
- `previous/`
|
||||||
- `reports/`
|
- `reports/`
|
||||||
- `logs/`
|
- `logs/`
|
||||||
- `config/`
|
- `config/`
|
||||||
@@ -99,6 +103,12 @@ Configured artifact source reuse:
|
|||||||
- accepted on `run`, `resume`, and `run-stage analyze`.
|
- accepted on `run`, `resume`, and `run-stage analyze`.
|
||||||
- filters analyze execution only; does not force stage rerun.
|
- filters analyze execution only; does not force stage rerun.
|
||||||
|
|
||||||
|
Canonical previous-session input behavior:
|
||||||
|
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
|
||||||
|
- these inputs are hydrated by `prepare`, not `analyze`.
|
||||||
|
- if analyze fails due to missing canonical previous cache, rerun:
|
||||||
|
- `narratio run-stage --session-id <id> --force prepare`
|
||||||
|
|
||||||
## Remote archive layout and publish contract
|
## Remote archive layout and publish contract
|
||||||
|
|
||||||
When archive is enabled and run upload is enabled, archive publishes under:
|
When archive is enabled and run upload is enabled, archive publishes under:
|
||||||
@@ -122,6 +132,8 @@ Archive promotion is explicit and source-based:
|
|||||||
- missing required promotion sources fail archive stage.
|
- missing required promotion sources fail archive stage.
|
||||||
- missing optional promotion sources are skipped.
|
- missing optional promotion sources are skipped.
|
||||||
- invalid resolved artifacts fail archive stage.
|
- invalid resolved artifacts fail archive stage.
|
||||||
|
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
|
||||||
|
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
|
||||||
|
|
||||||
## Resume, retry, restore, and safe rerun behavior
|
## Resume, retry, restore, and safe rerun behavior
|
||||||
|
|
||||||
@@ -139,6 +151,7 @@ Restore conflict policy:
|
|||||||
|
|
||||||
Forced reruns:
|
Forced reruns:
|
||||||
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
||||||
|
- ordinary `--force` does not override archive locks.
|
||||||
|
|
||||||
Safe rerun pattern:
|
Safe rerun pattern:
|
||||||
1. rerun the changed stage with `--force`.
|
1. rerun the changed stage with `--force`.
|
||||||
|
|||||||
318
docs/roadmap/remote.md
Normal file
318
docs/roadmap/remote.md
Normal file
@@ -0,0 +1,318 @@
|
|||||||
|
# Roadmap: Campaign State, Remote Sessions, and Archive Locks
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This roadmap tracks implementation work for three related feature areas:
|
||||||
|
|
||||||
|
1. `campaign.yml` configuration for stable campaign-level inputs.
|
||||||
|
2. Remote `session.yml` loading from the existing S3 object-store backend.
|
||||||
|
3. Logical archive locks that prevent selected top-level transcript/artifact promotions from overwriting curated archive state while still preserving run-local outputs.
|
||||||
|
|
||||||
|
Treat future sections of this document as an implementation plan, not as current behavior. Keep planned behavior under `docs/roadmap/` until each item is implemented and canonical docs are updated.
|
||||||
|
|
||||||
|
## Current Code Facts
|
||||||
|
|
||||||
|
The current codebase already settles several design choices:
|
||||||
|
|
||||||
|
- CLI commands use short noun flags: `--config`, `--session`, `--session-id`, `--previous-session-id`, `--force`, and `--artifacts`.
|
||||||
|
- `run-stage` uses flags before the positional stage name, for example:
|
||||||
|
|
||||||
|
narratio run-stage --session ./session.yml prepare
|
||||||
|
|
||||||
|
- `session.yml` is represented by `config.SessionConfig` and currently owns `session_id`, `previous_session_id`, `campaign`, `date`, `title`, and `inputs`.
|
||||||
|
- Strict YAML decoding is already implemented with `yaml.Decoder.KnownFields(true)`.
|
||||||
|
- Local session discovery is already ordered as `./session.yml`, `/usr/local/etc/narratio/session.yml`, then `/etc/narratio/session.yml`.
|
||||||
|
- The canonical S3 session prefix is already:
|
||||||
|
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/
|
||||||
|
|
||||||
|
- Archive promotion is already source-based through `archive.promote_artifacts[].source`, with destination derivation and validation in `internal/config`.
|
||||||
|
- Storage adapters receive bucket-relative keys and do not infer campaign, session, run, or root-prefix semantics.
|
||||||
|
|
||||||
|
## Guardrails
|
||||||
|
|
||||||
|
Keep Narratio explicit and stage-driven. Do not introduce a generic workflow engine, broad config language, or stage behavior that reaches through adapter boundaries.
|
||||||
|
|
||||||
|
Implementation must preserve these constraints:
|
||||||
|
|
||||||
|
- Keep storage details behind `internal/adapters/storage`.
|
||||||
|
- Compute session, campaign, archive, and remote config keys in app/artifact/path helpers, not inside storage implementations.
|
||||||
|
- Use centralized path helpers in `internal/artifacts` or the established local path model.
|
||||||
|
- Preserve manifest-driven resume and stage status semantics.
|
||||||
|
- Keep strict YAML decoding for `pipeline.yml`, `campaign.yml`, and `session.yml`.
|
||||||
|
- Keep raw secrets out of configs, manifests, logs, generated configs, archive metadata, and roadmap examples.
|
||||||
|
- Update canonical user-facing docs only after behavior is implemented.
|
||||||
|
|
||||||
|
## Phase 1: Add `campaign.yml` (implemented)
|
||||||
|
|
||||||
|
Add campaign-level configuration for stable campaign identity and stable input files. Do not add remote campaign loading in this phase.
|
||||||
|
|
||||||
|
### CLI and Discovery
|
||||||
|
|
||||||
|
Add `--campaign <path>` to `run`, `plan`, `resume`, `run-stage`, and `restore`.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
narratio run --campaign ./campaign.yml --session ./session.yml
|
||||||
|
narratio plan --campaign ./campaign.yml --session ./session.yml
|
||||||
|
narratio resume --campaign ./campaign.yml --session ./session.yml
|
||||||
|
narratio run-stage --campaign ./campaign.yml --session ./session.yml prepare
|
||||||
|
narratio restore --campaign ./campaign.yml --session ./session.yml
|
||||||
|
|
||||||
|
Campaign config discovery order:
|
||||||
|
|
||||||
|
1. explicit `--campaign <path>`;
|
||||||
|
2. `./campaign.yml`;
|
||||||
|
3. `/usr/local/etc/narratio/campaign.yml`;
|
||||||
|
4. `/etc/narratio/campaign.yml`.
|
||||||
|
|
||||||
|
Implement this in the same style as `resolvePipelineConfigPath` and `resolveSessionConfigPath`. Add default path constants and a search-path variable in `internal/config/defaults.go`.
|
||||||
|
|
||||||
|
### Config Shape
|
||||||
|
|
||||||
|
Initial `campaign.yml` fields:
|
||||||
|
|
||||||
|
campaign: icewind-dale
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
|
||||||
|
Do not add speculative campaign artifact defaults, prompt defaults, or title conventions in the first implementation.
|
||||||
|
|
||||||
|
### Merge Behavior
|
||||||
|
|
||||||
|
Add `CampaignConfig` and keep the final stage-facing config explicit.
|
||||||
|
|
||||||
|
Required behavior:
|
||||||
|
|
||||||
|
- `pipeline.yml` remains host/runtime configuration.
|
||||||
|
- `campaign.yml` supplies campaign identity and stable input file defaults.
|
||||||
|
- `session.yml` remains the source for `session_id`, `previous_session_id`, `date`, `title`, and audio input.
|
||||||
|
- Campaign-level `speakers_file`, `autocorrect_file`, and `glossary_file` fill missing session-level stable input fields.
|
||||||
|
- Session-level stable input fields override campaign-level stable input fields.
|
||||||
|
- If both `campaign.yml` and `session.yml` specify `campaign`, the values must match.
|
||||||
|
- The resolved session must satisfy the existing session validation rules before stages run.
|
||||||
|
- Unknown fields in `campaign.yml` fail strict decode.
|
||||||
|
|
||||||
|
Path resolution must preserve source-file locality:
|
||||||
|
|
||||||
|
- campaign-provided stable input paths resolve relative to `campaign.yml`;
|
||||||
|
- session-provided stable input overrides resolve relative to `session.yml`;
|
||||||
|
- absolute paths keep existing behavior.
|
||||||
|
|
||||||
|
Track enough provenance in the resolved config or prepare inputs so `prepare` can copy the correct source files without guessing which file supplied each path.
|
||||||
|
|
||||||
|
### Prepare Behavior
|
||||||
|
|
||||||
|
Update `prepare` to materialize the resolved campaign/session inputs into canonical session input paths:
|
||||||
|
|
||||||
|
inputs/campaign.yml
|
||||||
|
inputs/session.yml
|
||||||
|
inputs/pipeline.resolved.yml
|
||||||
|
inputs/speakers.yml
|
||||||
|
inputs/autocorrect.yml
|
||||||
|
inputs/glossary.yml
|
||||||
|
|
||||||
|
Continue recording deterministic `manifest.Inputs` records with checksums. If a prepared input came from `campaign.yml`, record source/provenance using the existing manifest input fields where practical; add narrow metadata only if the existing fields cannot describe it.
|
||||||
|
|
||||||
|
## Phase 2: Load Remote `session.yml` (implemented)
|
||||||
|
|
||||||
|
Support running with no local session file when a remote session file exists under the canonical session prefix.
|
||||||
|
|
||||||
|
### Preconditions
|
||||||
|
|
||||||
|
Build this phase after `campaign.yml`, because campaign identity is required to compute the remote session key. Do not infer campaign identity from object-store listing.
|
||||||
|
|
||||||
|
### Loading Precedence
|
||||||
|
|
||||||
|
Session loading order:
|
||||||
|
|
||||||
|
1. If `--session <path>` is supplied, load that local file.
|
||||||
|
2. If `--session` is omitted, use existing local discovery: `./session.yml`, `/usr/local/etc/narratio/session.yml`, `/etc/narratio/session.yml`.
|
||||||
|
3. If no local session file is found, `--session-id` is present, storage is configured, and campaign identity is resolved, load remote `session.yml`.
|
||||||
|
4. If no local or remote session can be loaded, fail with a message that lists the local search paths and the remote key that was attempted when applicable.
|
||||||
|
|
||||||
|
Do not make remote loading mask local discovery. Existing local discovery remains the local fallback before remote is attempted. Once remote loading is attempted, a missing remote object, storage init error, or malformed remote YAML fails clearly because no local session was available.
|
||||||
|
|
||||||
|
### Remote Key Layout
|
||||||
|
|
||||||
|
Use the existing canonical S3 layout:
|
||||||
|
|
||||||
|
session prefix: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
|
||||||
|
session file: {session_prefix}/session.yml
|
||||||
|
audio prefix: {session_prefix}/{session.inputs.audio_s3.prefix}
|
||||||
|
|
||||||
|
Add a centralized helper near `internal/artifacts/s3_keys.go`:
|
||||||
|
|
||||||
|
S3SessionConfigKey(sessionPrefix string) string
|
||||||
|
|
||||||
|
The helper should return `{session_prefix}/session.yml` using the same key normalization style as `S3CurrentManifestKey`, `S3CurrentRunPointerKey`, and `S3PromotedArtifactKey`.
|
||||||
|
|
||||||
|
### Decode, Template, and Provenance
|
||||||
|
|
||||||
|
Remote `session.yml` uses the same template variables and mismatch checks as local sessions:
|
||||||
|
|
||||||
|
- `{{session_id}}`
|
||||||
|
- `{{ session_id }}`
|
||||||
|
- `{{previous_session_id}}`
|
||||||
|
- `{{ previous_session_id }}`
|
||||||
|
|
||||||
|
Decode remote session YAML with strict known-field validation. Reuse the current session template/render/decode path by adding a byte/string-based loader rather than duplicating YAML decode logic.
|
||||||
|
|
||||||
|
When `prepare` materializes a remote session into `inputs/session.yml`, record that it came from S3. Preserve useful non-secret provenance when available:
|
||||||
|
|
||||||
|
- bucket;
|
||||||
|
- key;
|
||||||
|
- ETag;
|
||||||
|
- size;
|
||||||
|
- local checksum;
|
||||||
|
- downloaded temp/materialized path.
|
||||||
|
|
||||||
|
## Phase 3: Add Logical Archive Locks (implemented)
|
||||||
|
|
||||||
|
Narratio supports source-based archive locks under `pipeline.archive.locks`. The promotion system is source-based; destination-based locks are not supported.
|
||||||
|
|
||||||
|
### Config Shape
|
||||||
|
|
||||||
|
Lock entries:
|
||||||
|
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.polished
|
||||||
|
reason: Human-reviewed transcript; do not overwrite automatically.
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
reason: Final recap was manually edited.
|
||||||
|
|
||||||
|
Validation rules:
|
||||||
|
|
||||||
|
- `source` is required.
|
||||||
|
- `source` must be a built-in source ID or configured `narratio.artifact.<key>` accepted by the same source validation used for `promote_artifacts`.
|
||||||
|
- `reason` is optional and non-secret.
|
||||||
|
- duplicate lock sources fail validation.
|
||||||
|
- lock entries do not support `dest` in the first implementation; unknown fields already fail strict decode.
|
||||||
|
|
||||||
|
### Archive Behavior
|
||||||
|
|
||||||
|
Archive continues uploading complete run-local outputs under `runs/{run_id}/`.
|
||||||
|
|
||||||
|
Promotion behavior:
|
||||||
|
|
||||||
|
1. Resolve promotion source and destination using existing source-based promotion logic.
|
||||||
|
2. If the promotion source is unlocked, upload the top-level promoted object normally.
|
||||||
|
3. If the promotion source is locked, skip only the top-level promotion overwrite.
|
||||||
|
4. Treat locked required promotions as intentional successful skips by default.
|
||||||
|
5. Continue archive commit when all run-local uploads and all non-locked required promotions succeed.
|
||||||
|
6. Upload `current/manifest.json` and `current/run_id.txt` in the existing order, with `current/run_id.txt` last.
|
||||||
|
|
||||||
|
Ordinary `--force` does not override locks. A lock-break override remains out of scope.
|
||||||
|
|
||||||
|
### Metadata
|
||||||
|
|
||||||
|
Archive records locked promotion skips in archive metadata/reporting so operators can distinguish missing optional promotions from lock-protected promotions.
|
||||||
|
|
||||||
|
Include:
|
||||||
|
|
||||||
|
- source ID;
|
||||||
|
- destination relative path and remote key;
|
||||||
|
- reason;
|
||||||
|
- local resolved path;
|
||||||
|
- resolved provenance;
|
||||||
|
- whether the original promotion rule was required.
|
||||||
|
|
||||||
|
Keep existing metadata such as `promoted_paths`, `skipped_optional_promotions`, `current_manifest_key`, `current_run_id_key`, and `current_pointer_written`.
|
||||||
|
|
||||||
|
## Phase 4: Future Operator Helpers
|
||||||
|
|
||||||
|
These commands are future work only. Do not implement them with the first campaign, remote-session, or lock changes.
|
||||||
|
|
||||||
|
Potential helper shapes:
|
||||||
|
|
||||||
|
narratio session validate --session-id 2026-06-07
|
||||||
|
narratio session init --session-id 2026-06-07 --title "The Black Cabin"
|
||||||
|
narratio status --session-id 2026-06-07
|
||||||
|
narratio locks --session-id 2026-06-07
|
||||||
|
narratio lock narratio.artifact.session_recap --session-id 2026-06-07
|
||||||
|
narratio unlock narratio.artifact.session_recap --session-id 2026-06-07
|
||||||
|
|
||||||
|
Potential behavior:
|
||||||
|
|
||||||
|
- validate remote session config;
|
||||||
|
- check audio object availability;
|
||||||
|
- inspect committed remote current state;
|
||||||
|
- list promoted transcripts/artifacts;
|
||||||
|
- list archive lock status;
|
||||||
|
- initialize a remote session skeleton;
|
||||||
|
- publish or sync campaign assets.
|
||||||
|
|
||||||
|
## Implementation Sequence
|
||||||
|
|
||||||
|
Use small, reviewable commits.
|
||||||
|
|
||||||
|
1. Campaign config types and discovery:
|
||||||
|
add `CampaignConfig`, strict loading, defaults/search paths, `--campaign` flags, and config/app tests.
|
||||||
|
2. Campaign/session merge:
|
||||||
|
implement resolved stable input merge, path provenance, validation, and prepare materialization.
|
||||||
|
3. Campaign docs after implementation:
|
||||||
|
update canonical docs and examples only for implemented behavior.
|
||||||
|
4. Remote session key and loader:
|
||||||
|
add `S3SessionConfigKey`, byte/string session loading, remote download through `ObjectStore`, and app-level precedence tests.
|
||||||
|
5. Remote session prepare provenance:
|
||||||
|
materialize downloaded session config and record S3 provenance.
|
||||||
|
6. Remote session docs after implementation:
|
||||||
|
update canonical docs and examples only after behavior exists.
|
||||||
|
7. Archive lock config:
|
||||||
|
add lock config structs, strict decode coverage, source validation, and duplicate detection.
|
||||||
|
8. Archive lock enforcement:
|
||||||
|
skip locked top-level promotions, preserve run-local uploads, record lock metadata, and protect commit ordering.
|
||||||
|
9. Final sweep:
|
||||||
|
run focused tests, then `go test ./...`; verify planned behavior remains only in roadmap docs until implemented.
|
||||||
|
|
||||||
|
## Test Plan
|
||||||
|
|
||||||
|
Add focused coverage in these packages:
|
||||||
|
|
||||||
|
- `internal/config`: campaign load, strict decode, discovery constants, merge validation, campaign/session mismatch, lock validation, duplicate lock rejection.
|
||||||
|
- `internal/app`: `--campaign` parsing on `run`, `plan`, `resume`, `run-stage`, and `restore`; campaign discovery; explicit `--session` precedence; local discovery before remote; remote session fallback when local discovery misses.
|
||||||
|
- `internal/artifacts`: `S3SessionConfigKey`; canonical session prefix compatibility; source ID validation for lock sources.
|
||||||
|
- `internal/stage/prepare`: campaign/session stable input materialization; campaign-relative and session-relative path resolution; remote session provenance in `manifest.Inputs`.
|
||||||
|
- `internal/stage/archive`: locked required promotion succeeds as skipped; unlocked promotion uploads; run-local outputs upload when top-level promotion is locked; `--force` does not break locks; `current/run_id.txt` remains the last upload.
|
||||||
|
- `internal/adapters/storage`: fake object key normalization and remote session download expectations.
|
||||||
|
|
||||||
|
Run at least:
|
||||||
|
|
||||||
|
go test ./internal/config -v
|
||||||
|
go test ./internal/app -run TestExecute -v
|
||||||
|
go test ./internal/artifacts -v
|
||||||
|
go test ./internal/stage -run 'Prepare|Archive' -v
|
||||||
|
go test ./internal/adapters/storage -v
|
||||||
|
go test ./...
|
||||||
|
|
||||||
|
Use fake storage for remote-session and archive-lock behavior. Ordinary tests must not require live S3.
|
||||||
|
|
||||||
|
## Documentation Updates After Implementation
|
||||||
|
|
||||||
|
After each phase is implemented, update only docs for behavior that exists.
|
||||||
|
|
||||||
|
Likely files:
|
||||||
|
|
||||||
|
- `docs/config.md`
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/stage-prepare.md`
|
||||||
|
- `docs/internal/stage-archive.md`
|
||||||
|
- `docs/internal/storage.md`
|
||||||
|
- `docs/internal/artifacts.md`
|
||||||
|
- relevant examples under `examples/`
|
||||||
|
|
||||||
|
Do not document remote campaign loading, helper commands, or lock override flags as current behavior until implemented.
|
||||||
|
|
||||||
|
## Remaining Open Decisions
|
||||||
|
|
||||||
|
The codebase resolves the campaign flag name, campaign discovery order, remote session layout, local-vs-remote session precedence, source-based archive lock model, and locked required promotion policy.
|
||||||
|
|
||||||
|
Remaining decisions:
|
||||||
|
|
||||||
|
1. Whether `campaign.yml` should eventually be loadable from S3. Do not implement remote campaign loading in the first phase.
|
||||||
|
2. Whether a future explicit lock override command or flag is needed. Do not make ordinary `--force` break locks.
|
||||||
|
3. Whether future helper commands should be top-level commands or subcommands. Keep them out of the first implementation.
|
||||||
@@ -1,715 +0,0 @@
|
|||||||
# Roadmap: `narratio restore` Subcommand
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Implemented through Step 8. This document remains as roadmap and design history for the restore feature, and as the home for future restore-related ideas (for example `run --restore`).
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Add a new `narratio restore` subcommand that hydrates a local session workspace from the current committed remote archive state.
|
|
||||||
|
|
||||||
The primary operator workflow is:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio restore --session-id 2026-04-04
|
|
||||||
narratio run-stage --force analyze
|
|
||||||
```
|
|
||||||
|
|
||||||
This should allow a new machine with no local workspace state to restore the durable session manifest, transcripts, and generated artifacts from S3, then generate new Scriptorium artifacts without re-running transcription, merge, normalize, polish, or trim.
|
|
||||||
|
|
||||||
This is intentionally a separate command. Do not fold this behavior into the `prepare` stage. The existing `prepare` stage should remain focused on materializing configured local/S3 inputs for a pipeline run.
|
|
||||||
|
|
||||||
## Goals
|
|
||||||
|
|
||||||
- Add a first-class `narratio restore` command.
|
|
||||||
- Restore the current committed remote session state into the canonical local session workspace.
|
|
||||||
- Use the existing object storage adapter boundary.
|
|
||||||
- Preserve archive commit semantics: only restore from a remote state that has a valid current commit marker.
|
|
||||||
- Restore durable session-level outputs needed for downstream stages, especially `analyze`.
|
|
||||||
- Provide safe conflict behavior by default.
|
|
||||||
- Support `--dry-run`, `--force`, and `--include-audio`.
|
|
||||||
- Keep the implementation explicit, testable, and narrow.
|
|
||||||
|
|
||||||
## Non-goals
|
|
||||||
|
|
||||||
- Do not make `restore` a pipeline stage.
|
|
||||||
- Do not change the `prepare` stage behavior as part of this work.
|
|
||||||
- Do not add implicit restore behavior to `narratio run` in this implementation.
|
|
||||||
- Do not restore historical run-local sandboxes by default.
|
|
||||||
- Do not implement a generic remote synchronization engine.
|
|
||||||
- Do not implement bidirectional sync.
|
|
||||||
- Do not delete local files merely because they are absent remotely.
|
|
||||||
- Do not merge remote and local manifests in the first implementation.
|
|
||||||
- Do not require live S3 for the ordinary unit test suite.
|
|
||||||
|
|
||||||
## Future work explicitly out of scope
|
|
||||||
|
|
||||||
A future change may add:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run --restore
|
|
||||||
```
|
|
||||||
|
|
||||||
That future flag should run `narratio restore` before starting the normal pipeline. Mention this as future work in roadmap/docs if useful, but do not implement it now.
|
|
||||||
|
|
||||||
## Existing architecture to preserve
|
|
||||||
|
|
||||||
### `prepare` remains input materialization
|
|
||||||
|
|
||||||
The `prepare` stage currently materializes required session inputs into canonical local workspace paths and records input provenance. It owns local copying/materialization of config and audio inputs, including S3 audio download when `session.inputs.audio_s3.prefix` is configured. It does not own transcript generation/processing or archive publish behavior.
|
|
||||||
|
|
||||||
`restore` should not be implemented by expanding `prepare`. It should be an app-level command that reuses shared helpers where appropriate.
|
|
||||||
|
|
||||||
### Workspace model
|
|
||||||
|
|
||||||
The local durable session workspace is campaign-aware:
|
|
||||||
|
|
||||||
```text
|
|
||||||
{workspace.root}/work/{campaign}/{session_id}/
|
|
||||||
```
|
|
||||||
|
|
||||||
It contains durable session paths such as:
|
|
||||||
|
|
||||||
```text
|
|
||||||
manifest.json
|
|
||||||
inputs/
|
|
||||||
audio/
|
|
||||||
transcripts/
|
|
||||||
artifacts/
|
|
||||||
reports/
|
|
||||||
logs/
|
|
||||||
config/
|
|
||||||
current/
|
|
||||||
runs/
|
|
||||||
```
|
|
||||||
|
|
||||||
Run-local sandboxes live below:
|
|
||||||
|
|
||||||
```text
|
|
||||||
runs/{run_id}/
|
|
||||||
```
|
|
||||||
|
|
||||||
Restore should target durable session-level paths, not old run-local stage sandboxes.
|
|
||||||
|
|
||||||
### Storage boundary
|
|
||||||
|
|
||||||
The storage adapter owns object-store primitives only: `List`, `Download`, `Upload`, and `Exists`.
|
|
||||||
|
|
||||||
The storage adapter must not infer root prefixes, campaign names, session IDs, run IDs, or archive layout. Restore code must construct full bucket-relative keys before calling storage.
|
|
||||||
|
|
||||||
### Archive commit boundary
|
|
||||||
|
|
||||||
A remote run is current only after the archive stage has uploaded the run record, promoted outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
|
||||||
|
|
||||||
`current/run_id.txt` is the final remote commit marker and must be written last.
|
|
||||||
|
|
||||||
Restore must not treat incomplete, skipped, failed, or uncommitted archive attempts as current remote state.
|
|
||||||
|
|
||||||
## User-facing command
|
|
||||||
|
|
||||||
Add:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio restore [flags]
|
|
||||||
```
|
|
||||||
|
|
||||||
The command should use the same configuration/session discovery conventions as `run`, `plan`, `resume`, and `run-stage` where practical:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio restore --config /path/to/pipeline.yml --session ./session.yml --session-id 2026-04-04
|
|
||||||
```
|
|
||||||
|
|
||||||
Required effective inputs:
|
|
||||||
|
|
||||||
- resolved pipeline config;
|
|
||||||
- resolved session config;
|
|
||||||
- `session.campaign`;
|
|
||||||
- `session.session_id`;
|
|
||||||
- configured remote storage backend.
|
|
||||||
|
|
||||||
Supported flags:
|
|
||||||
|
|
||||||
```text
|
|
||||||
--config <path> Existing pipeline config path behavior.
|
|
||||||
--session <path> Existing session config path behavior.
|
|
||||||
--session-id <value> Existing session template behavior.
|
|
||||||
--dry-run Plan restore actions without writing local files.
|
|
||||||
--force Overwrite conflicting local files with remote files.
|
|
||||||
--include-audio Include archived session-level audio files.
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not add `--restore` to `run` in this implementation.
|
|
||||||
|
|
||||||
## Default restore scope
|
|
||||||
|
|
||||||
By default, restore:
|
|
||||||
|
|
||||||
1. Validates and reads the current remote commit marker.
|
|
||||||
2. Downloads the current remote manifest into the local session manifest path.
|
|
||||||
3. Downloads durable transcript files.
|
|
||||||
4. Downloads durable generated artifact files.
|
|
||||||
|
|
||||||
Default included remote/local durable paths:
|
|
||||||
|
|
||||||
```text
|
|
||||||
manifest.json from remote current manifest
|
|
||||||
transcripts/**
|
|
||||||
artifacts/**
|
|
||||||
```
|
|
||||||
|
|
||||||
Default excluded paths:
|
|
||||||
|
|
||||||
```text
|
|
||||||
audio/** unless --include-audio is passed
|
|
||||||
runs/** always excluded for this implementation
|
|
||||||
logs/** excluded for this implementation
|
|
||||||
reports/** excluded for this implementation unless needed for current manifest validation
|
|
||||||
config/** excluded for this implementation
|
|
||||||
inputs/** excluded for this implementation
|
|
||||||
current/** remote control metadata only; do not mirror blindly
|
|
||||||
```
|
|
||||||
|
|
||||||
If the existing archive implementation stores promoted files in a different remote layout, use the existing archive/path helpers and current archive semantics rather than inventing a parallel layout.
|
|
||||||
|
|
||||||
## Remote state discovery
|
|
||||||
|
|
||||||
Implement restore around the current committed archive state.
|
|
||||||
|
|
||||||
Expected algorithm:
|
|
||||||
|
|
||||||
1. Resolve pipeline/session config.
|
|
||||||
2. Ensure storage is configured.
|
|
||||||
3. Ensure local workspace layout exists.
|
|
||||||
4. Acquire the session lock.
|
|
||||||
5. Build the remote session archive prefix using the same helpers/policy used by archive code.
|
|
||||||
6. Check for the remote `current/run_id.txt` commit marker.
|
|
||||||
7. Read the committed run ID.
|
|
||||||
8. Download `current/manifest.json` to a temporary file.
|
|
||||||
9. Validate that the manifest is parseable and belongs to the requested campaign/session.
|
|
||||||
10. Build a restore plan from the committed remote state.
|
|
||||||
11. Execute the restore plan unless `--dry-run` is set.
|
|
||||||
12. Emit a concise summary.
|
|
||||||
|
|
||||||
Important: `current/run_id.txt` is the commit marker. Do not restore from a remote session prefix merely because files exist under `transcripts/` or `artifacts/`.
|
|
||||||
|
|
||||||
## Restore planning
|
|
||||||
|
|
||||||
Create a planning layer before writing files.
|
|
||||||
|
|
||||||
A restore plan entry should include at least:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type RestoreAction struct {
|
|
||||||
Kind RestoreActionKind
|
|
||||||
RemoteKey string
|
|
||||||
LocalPath string
|
|
||||||
Size int64
|
|
||||||
ETag string
|
|
||||||
ExistsLocal bool
|
|
||||||
SameLocal bool
|
|
||||||
Conflict bool
|
|
||||||
Reason string
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Suggested action kinds:
|
|
||||||
|
|
||||||
```text
|
|
||||||
download
|
|
||||||
skip_same
|
|
||||||
skip_missing_optional
|
|
||||||
conflict
|
|
||||||
```
|
|
||||||
|
|
||||||
The restore planner should be deterministic:
|
|
||||||
|
|
||||||
- sort remote objects by key;
|
|
||||||
- sort planned actions by local path or stable restore priority;
|
|
||||||
- write/report stable output for tests.
|
|
||||||
|
|
||||||
## Conflict and overwrite policy
|
|
||||||
|
|
||||||
Default behavior should be safe.
|
|
||||||
|
|
||||||
For each planned file:
|
|
||||||
|
|
||||||
```text
|
|
||||||
local absent:
|
|
||||||
download
|
|
||||||
|
|
||||||
local present and same as remote:
|
|
||||||
skip
|
|
||||||
|
|
||||||
local present and different:
|
|
||||||
conflict; fail restore unless --force is set
|
|
||||||
|
|
||||||
--force:
|
|
||||||
overwrite local conflicting files with remote versions
|
|
||||||
|
|
||||||
--dry-run:
|
|
||||||
do not write any files; report what would happen
|
|
||||||
```
|
|
||||||
|
|
||||||
The first implementation may use size and checksum/hash comparison where available. If remote ETag cannot be treated as a content hash, compare by downloading to a temporary file and hashing locally before deciding whether a local file is the same. Prefer correctness over assuming provider-specific ETag semantics.
|
|
||||||
|
|
||||||
Do not delete local files that are not present remotely.
|
|
||||||
|
|
||||||
## File writing and transactionality
|
|
||||||
|
|
||||||
Restore should avoid partial writes.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- download each remote object to a temporary file under the session workspace or OS temp dir;
|
|
||||||
- validate downloaded content where possible before replacing local files;
|
|
||||||
- create parent directories as needed;
|
|
||||||
- atomically rename/copy into place only after successful download;
|
|
||||||
- do not overwrite local files unless `--force` is set;
|
|
||||||
- if a later file fails, preserve already-restored files but return a failure summary;
|
|
||||||
- never corrupt an existing local manifest on failed manifest download/parse.
|
|
||||||
|
|
||||||
Manifest restore is especially sensitive:
|
|
||||||
|
|
||||||
- download remote `current/manifest.json` to a temporary file;
|
|
||||||
- parse and validate it;
|
|
||||||
- if no local manifest exists, install it;
|
|
||||||
- if a local manifest exists and is equivalent, skip;
|
|
||||||
- if a local manifest exists and differs, fail unless `--force` is set;
|
|
||||||
- with `--force`, replace the local manifest with the remote manifest after validation;
|
|
||||||
- do not attempt a manifest merge in the initial implementation.
|
|
||||||
|
|
||||||
## Manifest semantics
|
|
||||||
|
|
||||||
`restore` is not a pipeline run and should not mark stages as running/succeeded/failed.
|
|
||||||
|
|
||||||
The restored remote manifest becomes the local session manifest. That is what allows a subsequent command such as:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run-stage --force analyze
|
|
||||||
```
|
|
||||||
|
|
||||||
to see existing upstream stage state and canonical durable outputs.
|
|
||||||
|
|
||||||
Do not create a new run manifest for `restore`.
|
|
||||||
|
|
||||||
It is acceptable to write a restore diagnostic report outside the manifest, for example:
|
|
||||||
|
|
||||||
```text
|
|
||||||
reports/restore-latest.json
|
|
||||||
```
|
|
||||||
|
|
||||||
or a timestamped report, if that pattern fits the existing codebase. The report must not contain secrets.
|
|
||||||
|
|
||||||
## Local workspace locking
|
|
||||||
|
|
||||||
`restore` should acquire the same session lock used by ordinary pipeline operations before modifying session workspace state.
|
|
||||||
|
|
||||||
If the lock is held, fail fast with the same lock-conflict behavior used elsewhere.
|
|
||||||
|
|
||||||
`--dry-run` may still acquire the lock for consistency, but it is acceptable to avoid the lock if the codebase already has a clear read-only command pattern. Prefer safety and simplicity.
|
|
||||||
|
|
||||||
## Audio behavior
|
|
||||||
|
|
||||||
By default, do not restore audio.
|
|
||||||
|
|
||||||
If `--include-audio` is passed:
|
|
||||||
|
|
||||||
- restore archived durable session-level audio files only;
|
|
||||||
- do not use run-scoped spool paths;
|
|
||||||
- do not mutate or delete spool state;
|
|
||||||
- do not infer original `session.inputs.audio_s3.prefix` behavior;
|
|
||||||
- respect the same conflict/force/dry-run behavior used for transcripts/artifacts.
|
|
||||||
|
|
||||||
If the archive does not contain durable audio files, `--include-audio` should report that no archived audio was found rather than failing, unless the final implementation chooses to treat explicit audio restore as required. Prefer non-failure for absent archived audio unless tests or existing archive semantics suggest otherwise.
|
|
||||||
|
|
||||||
## Remote object selection
|
|
||||||
|
|
||||||
Prefer using manifest/artifact metadata when it reliably identifies durable outputs.
|
|
||||||
|
|
||||||
Also support listing committed durable archive prefixes so restore can retrieve all top-level session artifacts that may not yet be fully represented in manifest metadata.
|
|
||||||
|
|
||||||
The implementation should inspect existing archive code before choosing the final object-selection method. Do not duplicate archive path construction.
|
|
||||||
|
|
||||||
Recommended selection priority:
|
|
||||||
|
|
||||||
1. Remote current manifest path.
|
|
||||||
2. Durable promoted transcript/artifact outputs recorded in the manifest or archive metadata, if available.
|
|
||||||
3. Objects under committed durable `transcripts/` and `artifacts/` archive prefixes.
|
|
||||||
4. Objects under durable `audio/` only when `--include-audio` is passed.
|
|
||||||
|
|
||||||
Always exclude:
|
|
||||||
|
|
||||||
```text
|
|
||||||
runs/**
|
|
||||||
```
|
|
||||||
|
|
||||||
for the first implementation.
|
|
||||||
|
|
||||||
## Package and file organization
|
|
||||||
|
|
||||||
Expected areas to inspect and update:
|
|
||||||
|
|
||||||
```text
|
|
||||||
cmd/narratio/
|
|
||||||
internal/app/
|
|
||||||
internal/adapters/storage/
|
|
||||||
internal/artifacts/
|
|
||||||
internal/manifest/
|
|
||||||
docs/
|
|
||||||
examples/
|
|
||||||
```
|
|
||||||
|
|
||||||
Suggested implementation shape:
|
|
||||||
|
|
||||||
```text
|
|
||||||
internal/app/restore.go
|
|
||||||
internal/app/restore_test.go
|
|
||||||
|
|
||||||
internal/archive/restore/
|
|
||||||
planner.go
|
|
||||||
executor.go
|
|
||||||
report.go
|
|
||||||
keys.go
|
|
||||||
*_test.go
|
|
||||||
```
|
|
||||||
|
|
||||||
The exact package name may vary. Use whatever best fits the existing repository, but keep these boundaries clear:
|
|
||||||
|
|
||||||
- `internal/app` owns CLI command handling, config/session loading, lock acquisition, and wiring.
|
|
||||||
- Restore planning/execution owns remote key discovery, conflict detection, downloads, and reporting.
|
|
||||||
- `internal/adapters/storage` remains a transport boundary only.
|
|
||||||
- Workspace/path helpers remain centralized; do not scatter string concatenation.
|
|
||||||
|
|
||||||
If the repository already has an `internal/archive` or archive-stage helper package, prefer extending that rather than creating a conflicting package layout.
|
|
||||||
|
|
||||||
## CLI output
|
|
||||||
|
|
||||||
`narratio restore` should print a concise operator summary.
|
|
||||||
|
|
||||||
Example successful output:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Restored session archive for sample-campaign/2026-04-04
|
|
||||||
Remote run: 20260504T031500Z-a1b2c3
|
|
||||||
Downloaded: 4
|
|
||||||
Skipped unchanged: 2
|
|
||||||
Conflicts: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
Example dry run:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Restore plan for sample-campaign/2026-04-04
|
|
||||||
Remote run: 20260504T031500Z-a1b2c3
|
|
||||||
Would download: transcripts/processed.json
|
|
||||||
Would download: transcripts/trimmed.json
|
|
||||||
Would skip unchanged: artifacts/session_recap.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Example conflict:
|
|
||||||
|
|
||||||
```text
|
|
||||||
restore conflict: local artifacts/session_recap.md differs from remote archive; rerun with --force to overwrite
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not print transcript or artifact content.
|
|
||||||
|
|
||||||
## Error behavior
|
|
||||||
|
|
||||||
Fail clearly when:
|
|
||||||
|
|
||||||
- storage backend is not configured;
|
|
||||||
- S3 bucket/config is missing or invalid;
|
|
||||||
- remote current commit marker is missing;
|
|
||||||
- remote current manifest is missing;
|
|
||||||
- remote manifest is invalid;
|
|
||||||
- remote manifest does not match requested campaign/session;
|
|
||||||
- local file differs from remote and `--force` is not set;
|
|
||||||
- a required remote object download fails;
|
|
||||||
- a local path would escape the session workspace;
|
|
||||||
- a remote key maps to an unsafe local path.
|
|
||||||
|
|
||||||
Skip or report non-fatal conditions when:
|
|
||||||
|
|
||||||
- optional audio restore finds no archived audio;
|
|
||||||
- an included prefix has no objects;
|
|
||||||
- a local file already matches the remote file.
|
|
||||||
|
|
||||||
## Path safety
|
|
||||||
|
|
||||||
Every restored file must map to a safe path under the session root.
|
|
||||||
|
|
||||||
Validation rules:
|
|
||||||
|
|
||||||
- local restore paths must be relative to the session root;
|
|
||||||
- reject absolute paths;
|
|
||||||
- reject `..` traversal;
|
|
||||||
- reject paths that escape through symlinks if the codebase has symlink-safe path checks;
|
|
||||||
- do not restore remote keys directly without mapping/classification;
|
|
||||||
- do not mirror arbitrary remote keys.
|
|
||||||
|
|
||||||
## Testing plan
|
|
||||||
|
|
||||||
Add focused unit tests. Do not require live S3.
|
|
||||||
|
|
||||||
### CLI tests
|
|
||||||
|
|
||||||
Add or update `internal/app` command tests for:
|
|
||||||
|
|
||||||
- `narratio restore --help`;
|
|
||||||
- restore accepts `--config`, `--session`, and `--session-id`;
|
|
||||||
- restore accepts `--dry-run`;
|
|
||||||
- restore accepts `--force`;
|
|
||||||
- restore accepts `--include-audio`;
|
|
||||||
- restore fails when storage is not configured;
|
|
||||||
- restore does not run pipeline stages.
|
|
||||||
|
|
||||||
### Restore planner tests
|
|
||||||
|
|
||||||
Test:
|
|
||||||
|
|
||||||
- missing `current/run_id.txt` fails;
|
|
||||||
- missing `current/manifest.json` fails;
|
|
||||||
- invalid manifest fails;
|
|
||||||
- wrong campaign/session manifest fails;
|
|
||||||
- default scope includes manifest/transcripts/artifacts;
|
|
||||||
- default scope excludes audio/logs/reports/config/runs;
|
|
||||||
- `--include-audio` includes durable audio;
|
|
||||||
- run-local keys are excluded;
|
|
||||||
- keys are sorted deterministically;
|
|
||||||
- unsafe remote-to-local paths are rejected.
|
|
||||||
|
|
||||||
### Conflict policy tests
|
|
||||||
|
|
||||||
Test:
|
|
||||||
|
|
||||||
- absent local file downloads;
|
|
||||||
- matching local file skips;
|
|
||||||
- differing local file conflicts by default;
|
|
||||||
- `--force` overwrites conflicts;
|
|
||||||
- `--dry-run` writes nothing;
|
|
||||||
- partial failure does not corrupt an existing local manifest.
|
|
||||||
|
|
||||||
### Storage/fake tests
|
|
||||||
|
|
||||||
Use fake storage to simulate:
|
|
||||||
|
|
||||||
- object listing;
|
|
||||||
- object download;
|
|
||||||
- missing objects;
|
|
||||||
- download failures;
|
|
||||||
- metadata/ETag behavior.
|
|
||||||
|
|
||||||
### Workspace/lock tests
|
|
||||||
|
|
||||||
Test:
|
|
||||||
|
|
||||||
- session layout is created before restore;
|
|
||||||
- session lock conflict fails;
|
|
||||||
- restored files land under the expected campaign/session workspace;
|
|
||||||
- no files are written outside the session root.
|
|
||||||
|
|
||||||
### Follow-up command workflow test
|
|
||||||
|
|
||||||
Add at least one test that simulates:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio restore --session-id 2026-04-04
|
|
||||||
narratio run-stage --force analyze
|
|
||||||
```
|
|
||||||
|
|
||||||
The test does not need to run real Scriptorium. Use existing fake/stub behavior to verify that restored transcripts and manifest state are sufficient for analyze-stage input resolution.
|
|
||||||
|
|
||||||
## Documentation updates when implemented
|
|
||||||
|
|
||||||
When the feature is implemented, update current-behavior docs:
|
|
||||||
|
|
||||||
```text
|
|
||||||
docs/cli.md
|
|
||||||
docs/operations.md
|
|
||||||
docs/internal/storage.md or docs/internal/archive/restore.md
|
|
||||||
```
|
|
||||||
|
|
||||||
If the documentation set does not yet have an internal restore document, add one consistent with the existing internal-doc style:
|
|
||||||
|
|
||||||
```text
|
|
||||||
docs/internal/command-restore.md
|
|
||||||
```
|
|
||||||
|
|
||||||
or:
|
|
||||||
|
|
||||||
```text
|
|
||||||
docs/internal/archive-restore.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Do not document future `narratio run --restore` behavior outside `docs/roadmap/` until implemented.
|
|
||||||
|
|
||||||
## Implementation phases
|
|
||||||
|
|
||||||
### Phase 1: Audit existing archive and path helpers (completed)
|
|
||||||
|
|
||||||
Before coding behavior, inspect:
|
|
||||||
|
|
||||||
```text
|
|
||||||
internal/app/
|
|
||||||
internal/stage/archive*
|
|
||||||
internal/adapters/storage/
|
|
||||||
internal/artifacts/
|
|
||||||
internal/manifest/
|
|
||||||
docs/internal/stage-archive.md, if present
|
|
||||||
```
|
|
||||||
|
|
||||||
Determine:
|
|
||||||
|
|
||||||
- exact remote archive key layout;
|
|
||||||
- how root prefix/campaign/session are modeled;
|
|
||||||
- how current commit marker keys are built;
|
|
||||||
- how current manifest is uploaded;
|
|
||||||
- where promoted outputs are uploaded;
|
|
||||||
- whether helper functions already exist for remote archive keys;
|
|
||||||
- whether local workspace path helpers can safely map restore destinations.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- small code comments or internal helper selection;
|
|
||||||
- no large behavior change yet unless required by tests.
|
|
||||||
|
|
||||||
### Phase 2: Add CLI surface and command wiring (completed)
|
|
||||||
|
|
||||||
Add `narratio restore` command parsing.
|
|
||||||
|
|
||||||
Wire flags:
|
|
||||||
|
|
||||||
```text
|
|
||||||
--config
|
|
||||||
--session
|
|
||||||
--session-id
|
|
||||||
--dry-run
|
|
||||||
--force
|
|
||||||
--include-audio
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the existing config/session load path where practical.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- command exists;
|
|
||||||
- help output is sensible;
|
|
||||||
- command validates basic inputs;
|
|
||||||
- command returns a clear “not yet implemented” or calls an empty planner if phased commits are desired;
|
|
||||||
- CLI tests pass.
|
|
||||||
|
|
||||||
### Phase 3: Implement remote current-state discovery (completed)
|
|
||||||
|
|
||||||
Add restore code that:
|
|
||||||
|
|
||||||
- creates an object store from resolved config;
|
|
||||||
- builds remote current marker key;
|
|
||||||
- reads `current/run_id.txt`;
|
|
||||||
- reads/downloads `current/manifest.json`;
|
|
||||||
- validates manifest identity;
|
|
||||||
- returns remote current-state metadata.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- fake-storage tests for current-state discovery;
|
|
||||||
- no local file writes beyond temporary files.
|
|
||||||
|
|
||||||
### Phase 4: Implement restore planning (completed)
|
|
||||||
|
|
||||||
Build deterministic restore plans for default scope and `--include-audio`.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- plan lists manifest, transcript, artifact files;
|
|
||||||
- plan excludes run-local data;
|
|
||||||
- plan detects local same/conflict/missing states;
|
|
||||||
- dry-run output works;
|
|
||||||
- no real file overwrite yet except temp comparisons as needed.
|
|
||||||
|
|
||||||
### Phase 5: Implement restore execution (completed)
|
|
||||||
|
|
||||||
Execute the plan safely:
|
|
||||||
|
|
||||||
- create directories;
|
|
||||||
- download to temporary files;
|
|
||||||
- validate content where practical;
|
|
||||||
- atomically install files;
|
|
||||||
- enforce default conflict failure;
|
|
||||||
- support `--force`;
|
|
||||||
- preserve existing manifest unless safe to replace.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- restore works end-to-end against fake storage;
|
|
||||||
- failures are clear and do not corrupt existing local manifest.
|
|
||||||
|
|
||||||
### Phase 6: Add restore report and operator summary (completed)
|
|
||||||
|
|
||||||
Add concise stdout summary and optional JSON restore report if consistent with project diagnostics.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- user-friendly output;
|
|
||||||
- durable diagnostic report if implemented;
|
|
||||||
- no content leakage.
|
|
||||||
|
|
||||||
### Phase 7: Workflow integration test (completed)
|
|
||||||
|
|
||||||
Add a test for restoring a previous session and then forcing `analyze`.
|
|
||||||
|
|
||||||
Deliverable:
|
|
||||||
|
|
||||||
- restored manifest/transcripts/artifacts are sufficient for analyze input resolution;
|
|
||||||
- no upstream stages rerun;
|
|
||||||
- no reliance on live subprocesses or S3.
|
|
||||||
|
|
||||||
### Phase 8: Documentation update (completed)
|
|
||||||
|
|
||||||
Once implemented, update current-behavior docs and internal command docs.
|
|
||||||
|
|
||||||
Also leave future `narratio run --restore` in roadmap only.
|
|
||||||
|
|
||||||
## Definition of done
|
|
||||||
|
|
||||||
The feature is complete when:
|
|
||||||
|
|
||||||
- `narratio restore` exists and is documented.
|
|
||||||
- It uses the same config/session discovery semantics as other commands where practical.
|
|
||||||
- It requires configured remote storage.
|
|
||||||
- It restores only from a committed current archive state.
|
|
||||||
- It restores the current manifest, transcripts, and artifacts by default.
|
|
||||||
- It restores audio only with `--include-audio`.
|
|
||||||
- It excludes run-local sandboxes.
|
|
||||||
- It fails on local/remote conflicts by default.
|
|
||||||
- `--force` overwrites conflicts.
|
|
||||||
- `--dry-run` writes nothing.
|
|
||||||
- It uses fake storage in tests.
|
|
||||||
- It does not change `prepare` behavior.
|
|
||||||
- It does not implement `narratio run --restore`.
|
|
||||||
- It avoids AWS SDK leakage outside the storage adapter.
|
|
||||||
- It uses centralized path/key helpers rather than scattered string concatenation.
|
|
||||||
- `go test ./...` passes.
|
|
||||||
|
|
||||||
## Suggested test commands
|
|
||||||
|
|
||||||
Run focused tests first:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/app -run TestExecute -v
|
|
||||||
go test ./internal/adapters/storage -v
|
|
||||||
go test ./internal/artifacts -v
|
|
||||||
go test ./internal/manifest -v
|
|
||||||
```
|
|
||||||
|
|
||||||
Then run the full suite:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Suggested commit message
|
|
||||||
|
|
||||||
```text
|
|
||||||
Add restore subcommand roadmap
|
|
||||||
```
|
|
||||||
5
examples/campaign.yml
Normal file
5
examples/campaign.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
@@ -133,9 +133,7 @@ scriptorium:
|
|||||||
source: narratio.transcript.trimmed
|
source: narratio.transcript.trimmed
|
||||||
required: true
|
required: true
|
||||||
previous_recap:
|
previous_recap:
|
||||||
source: previous_session_artifact
|
source: narratio.previous_session.artifact.session_recap
|
||||||
artifact: session_recap
|
|
||||||
path: ""
|
|
||||||
required: false
|
required: false
|
||||||
vars:
|
vars:
|
||||||
session_id: true
|
session_id: true
|
||||||
|
|||||||
@@ -81,8 +81,7 @@ scriptorium:
|
|||||||
source: narratio.transcript.trimmed
|
source: narratio.transcript.trimmed
|
||||||
required: true
|
required: true
|
||||||
previous_recap:
|
previous_recap:
|
||||||
source: previous_session_artifact
|
source: narratio.previous_session.artifact.session_recap
|
||||||
artifact: session_recap
|
|
||||||
required: false
|
required: false
|
||||||
vars:
|
vars:
|
||||||
session_id: true
|
session_id: true
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
session_id: 2026-05-03
|
session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
date: 2026-05-03
|
date: 2026-05-03
|
||||||
title: Sample Session
|
title: Sample Session
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
session_id: 2026-05-03
|
session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
date: 2026-05-03
|
date: 2026-05-03
|
||||||
title: Sample Session
|
title: Sample Session
|
||||||
inputs:
|
inputs:
|
||||||
audio_s3:
|
audio_s3:
|
||||||
prefix: audio/
|
prefix: audio/
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -1,7 +1,3 @@
|
|||||||
session_id: "{{ session_id }}"
|
session_id: "{{ session_id }}"
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./examples/speakers.yml
|
|
||||||
autocorrect_file: ./examples/autocorrect.yml
|
|
||||||
glossary_file: ./examples/glossary.yml
|
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ func TestFakeBackendDownload(t *testing.T) {
|
|||||||
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||||
|
|
||||||
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||||
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
|
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||||
t.Fatalf("Download() error = %v", err)
|
t.Fatalf("Download() error = %v", err)
|
||||||
}
|
}
|
||||||
data, err := os.ReadFile(dst)
|
data, err := os.ReadFile(dst)
|
||||||
|
|||||||
@@ -14,12 +14,12 @@ import (
|
|||||||
|
|
||||||
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
[]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap", "polish"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -33,12 +33,12 @@ func TestExecuteRunStageArtifactsNonAnalyzeFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run", "--config", pipelinePath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
[]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -52,7 +52,7 @@ func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -65,7 +65,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(
|
err := RunStage(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap", "analyze"},
|
||||||
&out,
|
&out,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -78,7 +78,7 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
|||||||
|
|
||||||
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -93,7 +93,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(
|
err := Resume(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
[]string{"--config", pipelinePath, "--session", sessionPath, "--artifacts", "session_recap"},
|
[]string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
&out,
|
&out,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -104,10 +104,10 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string) {
|
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open pipeline config for append: %v", err)
|
t.Fatalf("open pipeline config for append: %v", err)
|
||||||
@@ -136,5 +136,5 @@ scriptorium:
|
|||||||
if _, err := f.WriteString(extra); err != nil {
|
if _, err := f.WriteString(extra); err != nil {
|
||||||
t.Fatalf("append scriptorium config: %v", err)
|
t.Fatalf("append scriptorium config: %v", err)
|
||||||
}
|
}
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
}
|
}
|
||||||
|
|||||||
49
internal/app/campaign_config_path.go
Normal file
49
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolveCampaignConfigPath(flagValue string) (string, error) {
|
||||||
|
return resolveCampaignConfigPathWithCandidates(flagValue, config.DefaultCampaignConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveCampaignConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||||
|
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||||
|
return explicit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ordered := make([]string, 0, len(candidates))
|
||||||
|
for _, raw := range candidates {
|
||||||
|
path := strings.TrimSpace(raw)
|
||||||
|
if path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ordered = append(ordered, path)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
if info.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("check default campaign config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ordered) == 0 {
|
||||||
|
return "", fmt.Errorf("no campaign config path provided and no default locations configured")
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"no campaign config path provided and no default campaign config found; searched: %s; pass --campaign to use an explicit path",
|
||||||
|
strings.Join(ordered, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
49
internal/app/campaign_config_path_test.go
Normal file
49
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathExplicitWins(t *testing.T) {
|
||||||
|
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||||
|
got, err := resolveCampaignConfigPathWithCandidates(explicit, []string{filepath.Join(t.TempDir(), "campaign.yml")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != explicit {
|
||||||
|
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathUsesFirstExistingDefault(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
missing := filepath.Join(dir, "missing.yml")
|
||||||
|
found := filepath.Join(dir, "campaign.yml")
|
||||||
|
if err := os.WriteFile(found, []byte("campaign: sample-campaign\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolveCampaignConfigPathWithCandidates("", []string{missing, found})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(found) {
|
||||||
|
t.Fatalf("path = %q, want %q", got, filepath.Clean(found))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||||
|
_, err := resolveCampaignConfigPathWithCandidates("", []string{"./campaign.yml", "/usr/local/etc/narratio/campaign.yml", "/etc/narratio/campaign.yml"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "searched") {
|
||||||
|
t.Fatalf("error = %q, want searched paths", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "pass --campaign") {
|
||||||
|
t.Fatalf("error = %q, want explicit-campaign guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := writeManifestPathForExecute(t)
|
manifestPath := writeManifestPathForExecute(t)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -32,11 +32,11 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -94,12 +94,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -110,14 +110,14 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -136,19 +136,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
stdout.Reset()
|
stdout.Reset()
|
||||||
stderr.Reset()
|
stderr.Reset()
|
||||||
|
|
||||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
code = Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -188,6 +188,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
|||||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||||
|
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
@@ -239,7 +240,7 @@ inputs:
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -252,6 +253,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -288,7 +290,7 @@ inputs:
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"run", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -305,11 +307,14 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||||
|
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||||
|
config.DefaultCampaignConfigSearchPaths = []string{campaignPath}
|
||||||
defer func() {
|
defer func() {
|
||||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||||
|
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||||
}()
|
}()
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
@@ -323,6 +328,33 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteMissingCampaignConfigReportsSearchedPaths(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
missingCampaignPath := filepath.Join(t.TempDir(), "campaign.yml")
|
||||||
|
originalCampaignDefaults := append([]string(nil), config.DefaultCampaignConfigSearchPaths...)
|
||||||
|
config.DefaultCampaignConfigSearchPaths = []string{missingCampaignPath}
|
||||||
|
defer func() {
|
||||||
|
config.DefaultCampaignConfigSearchPaths = originalCampaignDefaults
|
||||||
|
}()
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "no campaign config path provided and no default campaign config found; searched:") {
|
||||||
|
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "pass --campaign") {
|
||||||
|
t.Fatalf("stderr = %q, want explicit campaign guidance", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteInvalidCommand(t *testing.T) {
|
func TestExecuteInvalidCommand(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -359,11 +391,12 @@ func TestExecuteMissingCommand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
url := "https://example.com/transcribe"
|
url := "https://example.com/transcribe"
|
||||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||||
@@ -407,9 +440,11 @@ notification:
|
|||||||
`
|
`
|
||||||
|
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
|
`
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
@@ -418,6 +453,9 @@ inputs:
|
|||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline config: %v", err)
|
t.Fatalf("write pipeline config: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign config: %v", err)
|
||||||
|
}
|
||||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write session config: %v", err)
|
t.Fatalf("write session config: %v", err)
|
||||||
}
|
}
|
||||||
@@ -427,7 +465,22 @@ inputs:
|
|||||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||||
|
t.Helper()
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
return campaignPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeManifestPathForExecute(t *testing.T) string {
|
func writeManifestPathForExecute(t *testing.T) string {
|
||||||
|
|||||||
133
internal/app/config_loader.go
Normal file
133
internal/app/config_loader.go
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvedCampaignPath, err := resolveCampaignConfigPath(campaignFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||||
|
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, explicitSession, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if discoveredSession.Path != "" {
|
||||||
|
return config.LoadWithSessionOptions(resolvedPipelinePath, resolvedCampaignPath, discoveredSession.Path, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires --session-id")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(pipelineCfg.Storage.S3.RootPrefix, campaignCfg.Campaign, sessionID)
|
||||||
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
partialCfg := &config.Config{
|
||||||
|
Pipeline: pipelineCfg,
|
||||||
|
Campaign: campaignCfg,
|
||||||
|
PipelinePath: resolvedPipelinePath,
|
||||||
|
CampaignPath: resolvedCampaignPath,
|
||||||
|
}
|
||||||
|
store, err := newObjectStoreFromConfigFn(ctx, partialCfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||||
|
}
|
||||||
|
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||||
|
}
|
||||||
|
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(pipelineCfg)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.Resolve(
|
||||||
|
resolvedPipelinePath,
|
||||||
|
pipelineCfg,
|
||||||
|
resolvedCampaignPath,
|
||||||
|
campaignCfg,
|
||||||
|
sessionTempPath,
|
||||||
|
sessionCfg,
|
||||||
|
config.SessionSource{
|
||||||
|
Source: "session_config.s3",
|
||||||
|
LocalPath: sessionTempPath,
|
||||||
|
S3Bucket: s3BucketName(pipelineCfg),
|
||||||
|
S3Key: remoteKey,
|
||||||
|
S3Size: sessionInfo.Size,
|
||||||
|
S3ETag: sessionInfo.ETag,
|
||||||
|
SpoolPath: sessionTempPath,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||||
|
objects, err := store.List(ctx, sessionPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||||
|
}
|
||||||
|
for _, obj := range objects {
|
||||||
|
if obj.Key == remoteKey {
|
||||||
|
return obj, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
|
||||||
|
f, err := os.CreateTemp("", "narratio-session-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := f.Name()
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("close temp file %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := store.Download(ctx, remoteKey, path); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||||
|
if cfg == nil || cfg.Storage.S3 == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||||
|
}
|
||||||
@@ -20,12 +20,16 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -34,17 +38,9 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("plan: unexpected positional arguments")
|
return fmt.Errorf("plan: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("plan: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("plan: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
|
|||||||
@@ -15,10 +15,10 @@ import (
|
|||||||
|
|
||||||
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
args := []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}
|
||||||
|
|
||||||
if err := Plan(context.Background(), args, &out); err != nil {
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
t.Fatalf("first Plan() error = %v", err)
|
t.Fatalf("first Plan() error = %v", err)
|
||||||
@@ -62,7 +62,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
|
|
||||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
|
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out); err != nil {
|
||||||
t.Fatalf("Plan() error = %v", err)
|
t.Fatalf("Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
got := out.String()
|
got := out.String()
|
||||||
@@ -93,6 +93,7 @@ func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
sessionPath := filepath.Join(configDir, "session.yml")
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -128,7 +129,7 @@ inputs:
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
|
|||||||
|
|
||||||
assertExists(t, cfg.Pipeline.Workspace.Root)
|
assertExists(t, cfg.Pipeline.Workspace.Root)
|
||||||
assertExists(t, seed.otherRunDir)
|
assertExists(t, seed.otherRunDir)
|
||||||
|
assertExists(t, seed.previousCachePath)
|
||||||
assertMissing(t, seed.runWorkDir)
|
assertMissing(t, seed.runWorkDir)
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
}
|
}
|
||||||
@@ -98,6 +99,7 @@ func TestPostArchiveCleanupBothPolicies(t *testing.T) {
|
|||||||
assertMissing(t, seed.spoolAudioDir)
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
assertMissing(t, seed.runWorkDir)
|
assertMissing(t, seed.runWorkDir)
|
||||||
assertExists(t, seed.otherRunDir)
|
assertExists(t, seed.otherRunDir)
|
||||||
|
assertExists(t, seed.previousCachePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
||||||
@@ -260,6 +262,7 @@ type cleanupSeed struct {
|
|||||||
otherRunDir string
|
otherRunDir string
|
||||||
spoolAudioDir string
|
spoolAudioDir string
|
||||||
localSourceAudio string
|
localSourceAudio string
|
||||||
|
previousCachePath string
|
||||||
sessionPrefix string
|
sessionPrefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -274,11 +277,18 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
||||||
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
|
previousCachePath := artifacts.SessionPreviousArtifactPathForCampaign(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
"session_recap.md",
|
||||||
|
)
|
||||||
|
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
||||||
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
||||||
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
||||||
|
mustWriteFile(t, previousCachePath, "# previous recap\n")
|
||||||
|
|
||||||
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
|
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
|
||||||
mustWriteFile(t, localSourceAudio, "source\n")
|
mustWriteFile(t, localSourceAudio, "source\n")
|
||||||
@@ -305,6 +315,7 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
otherRunDir: otherRunDir,
|
otherRunDir: otherRunDir,
|
||||||
spoolAudioDir: spoolAudioDir,
|
spoolAudioDir: spoolAudioDir,
|
||||||
localSourceAudio: localSourceAudio,
|
localSourceAudio: localSourceAudio,
|
||||||
|
previousCachePath: previousCachePath,
|
||||||
sessionPrefix: seed.S3SessionPrefix,
|
sessionPrefix: seed.S3SessionPrefix,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
204
internal/app/remote_session_test.go
Normal file
204
internal/app/remote_session_test.go
Normal file
@@ -0,0 +1,204 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 1 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio plan: workdir prepared") {
|
||||||
|
t.Fatalf("stdout = %q, want plan output", stdout.String())
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[remoteKey]; !ok {
|
||||||
|
t.Fatalf("remote session key %q was not seeded", remoteKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, config.DefaultSessionConfigSearchPaths)
|
||||||
|
|
||||||
|
originalWD, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Getwd(): %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(filepath.Dir(sessionPath)); err != nil {
|
||||||
|
t.Fatalf("Chdir(%q): %v", filepath.Dir(sessionPath), err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
var storeInitCalls int
|
||||||
|
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
|
||||||
|
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), missingSessionPath) {
|
||||||
|
t.Fatalf("stderr = %q, want local searched path", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "remote session loading requires --session-id") {
|
||||||
|
t.Fatalf("stderr = %q, want session-id guidance", stderr.String())
|
||||||
|
}
|
||||||
|
if storeInitCalls != 0 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
|
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
return nil, errors.New("storage unavailable")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||||
|
})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
|
||||||
|
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"plan", "--config", pipelinePath, "--campaign", campaignPath, "--session-id", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "strict decode failed") {
|
||||||
|
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
|
||||||
|
t.Helper()
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||||
|
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if storeInitCalls != nil {
|
||||||
|
(*storeInitCalls)++
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
|
||||||
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
fake.SeedObject(storage.FakeObject{
|
||||||
|
Key: remoteKey,
|
||||||
|
Data: []byte(content),
|
||||||
|
ETag: "remote-session-etag",
|
||||||
|
})
|
||||||
|
return remoteKey
|
||||||
|
}
|
||||||
@@ -26,19 +26,23 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(out)
|
fs.SetOutput(out)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
var dryRun bool
|
var dryRun bool
|
||||||
var force bool
|
var force bool
|
||||||
var includeAudio bool
|
var includeAudio bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
fs.BoolVar(&dryRun, "dry-run", false, "plan restore actions without writing local files")
|
fs.BoolVar(&dryRun, "dry-run", false, "plan restore actions without writing local files")
|
||||||
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
||||||
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
||||||
fs.Usage = func() {
|
fs.Usage = func() {
|
||||||
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--session <path>] [--session-id <value>] [--dry-run] [--force] [--include-audio]")
|
_, _ = fmt.Fprintln(out, "Usage: narratio restore [--config <path>] [--campaign <path>] [--session <path>] [--session-id <value>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
||||||
_, _ = fmt.Fprintln(out)
|
_, _ = fmt.Fprintln(out)
|
||||||
_, _ = fmt.Fprintln(out, "Flags:")
|
_, _ = fmt.Fprintln(out, "Flags:")
|
||||||
fs.PrintDefaults()
|
fs.PrintDefaults()
|
||||||
@@ -53,17 +57,9 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("restore: unexpected positional arguments")
|
return fmt.Errorf("restore: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
|
|||||||
@@ -18,10 +18,10 @@ import (
|
|||||||
|
|
||||||
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`))
|
||||||
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
|
seedRestoreObject(fake, sessionPrefix+"artifacts/session_recap.md", []byte("# recap\n"))
|
||||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||||
@@ -32,7 +32,7 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -64,17 +64,17 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
seedRestoreObject(fake, sessionPrefix+"audio/alice.flac", []byte("remote-audio"))
|
||||||
|
|
||||||
restoreWithStoreAndRealPhases(t, fake)
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--include-audio"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -87,12 +87,39 @@ func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
func TestExecuteRestoreRestoresPreviousCacheWhenPresent(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
|
seedRestoreObject(fake, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
||||||
|
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
||||||
|
|
||||||
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "manifest.json"), `{"session_id":"2026-04-26"}`)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
|
||||||
|
report := mustReadRestoreReport(t, filepath.Join(sessionRoot, "reports", "restore-latest.json"))
|
||||||
|
if report.Execution.Downloaded != 3 {
|
||||||
|
t.Fatalf("report execution.downloaded = %d, want 3", report.Execution.Downloaded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -102,7 +129,7 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -121,10 +148,10 @@ func TestExecuteRestoreConflictWithoutForceDoesNotOverwrite(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -134,7 +161,7 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -145,12 +172,34 @@ func TestExecuteRestoreForceOverwritesDifferingFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
func TestExecuteRestoreForceOverwritesDifferingPreviousCacheFile(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
|
seedRestoreObject(fake, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# remote previous recap\n"))
|
||||||
|
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustWriteTestFile(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# local previous recap\n")
|
||||||
|
|
||||||
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# remote previous recap\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
cfg, sessionPrefix, _, _ := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspaceRoot)
|
store := artifacts.NewLocalStore(workspaceRoot)
|
||||||
@@ -164,7 +213,7 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -180,10 +229,10 @@ func TestExecuteRestoreLockConflictFailsAndWritesNothing(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
|
func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
base := &storage.FakeBackend{}
|
base := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, _ := seedRestoreCommittedState(t, base, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
seedRestoreObject(base, sessionPrefix+"transcripts/full.json", []byte("remote-transcript"))
|
||||||
|
|
||||||
toggled := &stagedManifestDownloadStore{
|
toggled := &stagedManifestDownloadStore{
|
||||||
@@ -211,7 +260,7 @@ func TestExecuteRestoreInvalidManifestDoesNotCorruptExistingManifest(t *testing.
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -295,9 +344,9 @@ func restoreWithStoreAndRealPhases(t *testing.T, objectStore storage.ObjectStore
|
|||||||
executeRestorePlanFn = executeRestorePlan
|
executeRestorePlanFn = executeRestorePlan
|
||||||
}
|
}
|
||||||
|
|
||||||
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, sessionPath string) (*config.Config, string, string, string) {
|
func seedRestoreCommittedState(t *testing.T, fake *storage.FakeBackend, pipelinePath, campaignPath, sessionPath string) (*config.Config, string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, sessionPath, config.SessionLoadOptions{})
|
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -194,6 +194,9 @@ func restoreLocalRelativePathForKey(sessionPrefix, currentManifestKey, key strin
|
|||||||
if cleanRel == config.PathArtifactsDirSegment || strings.HasPrefix(cleanRel, config.PathArtifactsDirSegment+"/") {
|
if cleanRel == config.PathArtifactsDirSegment || strings.HasPrefix(cleanRel, config.PathArtifactsDirSegment+"/") {
|
||||||
return cleanRel, true, nil
|
return cleanRel, true, nil
|
||||||
}
|
}
|
||||||
|
if cleanRel == config.PathPreviousDirSegment || strings.HasPrefix(cleanRel, config.PathPreviousDirSegment+"/") {
|
||||||
|
return cleanRel, true, nil
|
||||||
|
}
|
||||||
if includeAudio && (cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")) {
|
if includeAudio && (cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")) {
|
||||||
return cleanRel, true, nil
|
return cleanRel, true, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,6 +58,27 @@ func TestRestorePlanIncludeAudio(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestorePlanIncludesPreviousCacheByDefault(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
current := restorePlanCurrentState(t, cfg)
|
||||||
|
store := &storage.FakeBackend{}
|
||||||
|
|
||||||
|
seedRestoreObject(store, current.CurrentManifestKey, []byte(`{"session_id":"2026-05-03"}`))
|
||||||
|
seedRestoreObject(store, current.SessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
||||||
|
seedRestoreObject(store, current.SessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
||||||
|
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := actionRelPaths(plan.Actions)
|
||||||
|
want := []string{"manifest.json", "previous/artifacts/session_recap.md", "previous/manifest.json"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("action local paths = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRestorePlanClassifiesSameAndConflict(t *testing.T) {
|
func TestRestorePlanClassifiesSameAndConflict(t *testing.T) {
|
||||||
cfg := restorePlanConfig(t)
|
cfg := restorePlanConfig(t)
|
||||||
current := restorePlanCurrentState(t, cfg)
|
current := restorePlanCurrentState(t, cfg)
|
||||||
|
|||||||
@@ -32,6 +32,9 @@ func TestExecuteRestoreHelp(t *testing.T) {
|
|||||||
if !strings.Contains(out, "--include-audio") {
|
if !strings.Contains(out, "--include-audio") {
|
||||||
t.Fatalf("stdout = %q, want --include-audio flag", out)
|
t.Fatalf("stdout = %q, want --include-audio flag", out)
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(out, "--campaign") {
|
||||||
|
t.Fatalf("stdout = %q, want --campaign flag", out)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
||||||
@@ -70,7 +73,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -78,6 +81,7 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", "2026-05-03",
|
"--session-id", "2026-05-03",
|
||||||
"--dry-run",
|
"--dry-run",
|
||||||
@@ -116,11 +120,11 @@ func TestExecuteRestoreRecognizedAndReturnsNYI(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
|
func TestExecuteRestoreRejectsUnexpectedPositionalArguments(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "extra"}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -138,11 +142,11 @@ func TestExecuteRestoreFailsWhenStorageBackendNotConfigured(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeRestoreConfigWithoutStorage(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -166,11 +170,11 @@ func TestExecuteRestoreDiscoveryErrorSurfaced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -208,7 +212,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
restoreEnv(secretKeyEnv)
|
restoreEnv(secretKeyEnv)
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
||||||
@@ -261,6 +265,7 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", "2026-05-03",
|
"--session-id", "2026-05-03",
|
||||||
"--dry-run",
|
"--dry-run",
|
||||||
@@ -307,10 +312,10 @@ func TestExecuteRestoreNonDryRunConflictFailsBeforeNYI(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -359,10 +364,10 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"restore", "--config", pipelinePath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"restore", "--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -374,11 +379,12 @@ func TestExecuteRestoreNonDryRunForceExecutesPlan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string) {
|
func writeRestoreConfigWithoutStorage(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, dir)
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
@@ -405,5 +411,5 @@ inputs:
|
|||||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,10 +19,10 @@ import (
|
|||||||
|
|
||||||
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, sessionPath)
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||||
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
seedRestoreObject(fake, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||||
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
|
seedRestoreObject(fake, sessionPrefix+"transcripts/full.json", []byte(`{"segments":[1,2,3]}`+"\n"))
|
||||||
@@ -48,6 +48,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"restore",
|
"restore",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
},
|
},
|
||||||
@@ -86,6 +87,7 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
[]string{
|
[]string{
|
||||||
"run-stage",
|
"run-stage",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
"--session-id", cfg.Session.SessionID,
|
"--session-id", cfg.Session.SessionID,
|
||||||
"--force",
|
"--force",
|
||||||
@@ -155,6 +157,118 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
appendRestoreWorkflowScriptoriumConfig(t, pipelinePath, `
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.trimmed
|
||||||
|
required: true
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session_recap
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
|
||||||
|
fakeStore := &storage.FakeBackend{}
|
||||||
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fakeStore, pipelinePath, campaignPath, sessionPath)
|
||||||
|
seedRestoreObject(fakeStore, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||||
|
seedRestoreObject(fakeStore, manifestKey, restoreWorkflowManifestJSON(t, cfg.Session.SessionID, cfg.Session.Campaign))
|
||||||
|
seedRestoreObject(fakeStore, sessionPrefix+"transcripts/trimmed.json", []byte(`{"segments":[]}`+"\n"))
|
||||||
|
seedRestoreObject(fakeStore, sessionPrefix+"previous/manifest.json", []byte(`{"session_id":"2026-04-26"}`))
|
||||||
|
seedRestoreObject(fakeStore, sessionPrefix+"previous/artifacts/session_recap.md", []byte("# previous recap\n"))
|
||||||
|
|
||||||
|
restoreWithStoreAndRealPhases(t, fakeStore)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
restoreCode := Execute(
|
||||||
|
[]string{
|
||||||
|
"restore",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", cfg.Session.SessionID,
|
||||||
|
},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if restoreCode != 0 {
|
||||||
|
t.Fatalf("restore exit code = %d, want 0; stderr=%q", restoreCode, stderr.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("restore stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), `{"segments":[]}`+"\n")
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "manifest.json"), `{"session_id":"2026-04-26"}`)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
|
||||||
|
|
||||||
|
scriptoriumFake := &scriptorium.FakeRunner{}
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
origObjectStoreFn := newObjectStoreFromConfigFn
|
||||||
|
objectStoreConstructed := false
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
newObjectStoreFromConfigFn = origObjectStoreFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(ctx context.Context, cfg *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
if opts.Env == nil {
|
||||||
|
opts.Env = &Env{}
|
||||||
|
}
|
||||||
|
opts.Env.Scriptorium = scriptoriumFake
|
||||||
|
return executeStages(ctx, cfg, stages, opts)
|
||||||
|
}
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
objectStoreConstructed = true
|
||||||
|
return nil, context.Canceled
|
||||||
|
}
|
||||||
|
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
runStageCode := Execute(
|
||||||
|
[]string{
|
||||||
|
"run-stage",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", cfg.Session.SessionID,
|
||||||
|
"--force",
|
||||||
|
"--artifacts", "session_recap",
|
||||||
|
"analyze",
|
||||||
|
},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if runStageCode != 0 {
|
||||||
|
t.Fatalf("run-stage exit code = %d, want 0; stderr=%q", runStageCode, stderr.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("run-stage stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
if objectStoreConstructed {
|
||||||
|
t.Fatal("analyze run-stage should not construct object store for previous-session input resolution")
|
||||||
|
}
|
||||||
|
if len(scriptoriumFake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("scriptorium run requests = %d, want 1", len(scriptoriumFake.RunRequests))
|
||||||
|
}
|
||||||
|
req := scriptoriumFake.RunRequests[0]
|
||||||
|
if got := req.InputPaths["transcript"]; got != filepath.Join(sessionRoot, "transcripts", "trimmed.json") {
|
||||||
|
t.Fatalf("transcript input = %q, want trimmed transcript path", got)
|
||||||
|
}
|
||||||
|
if got := req.InputPaths["previous_recap"]; got != filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md") {
|
||||||
|
t.Fatalf("previous_recap input = %q, want restored previous cache path", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func restoreWorkflowManifestJSON(t *testing.T, sessionID, campaign string) []byte {
|
func restoreWorkflowManifestJSON(t *testing.T, sessionID, campaign string) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -176,3 +290,15 @@ func restoreWorkflowManifestJSON(t *testing.T, sessionID, campaign string) []byt
|
|||||||
}
|
}
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func appendRestoreWorkflowScriptoriumConfig(t *testing.T, pipelinePath, extra string) {
|
||||||
|
t.Helper()
|
||||||
|
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open pipeline config for append: %v", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if _, err := f.WriteString(extra); err != nil {
|
||||||
|
t.Fatalf("append pipeline config: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,13 +17,17 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution")
|
fs.BoolVar(&force, "force", false, "force stage execution")
|
||||||
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
||||||
|
|
||||||
@@ -33,17 +37,9 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("resume: unexpected positional arguments")
|
return fmt.Errorf("resume: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("resume: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("resume: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("resume: %w", err)
|
return fmt.Errorf("resume: %w", err)
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -32,7 +32,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -51,7 +51,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
|
|
||||||
func TestResumeNoRemainingStages(t *testing.T) {
|
func TestResumeNoRemainingStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -64,7 +64,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -80,7 +80,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
|
_, _ = w.Write([]byte(`{"source":"resume-force-test","segments":[{"speaker":"alice"}]}`))
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -93,7 +93,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force"}, &out)
|
err := Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -104,14 +104,14 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -134,7 +134,7 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageSkipAndForce(t *testing.T) {
|
func TestRunStageSkipAndForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
@@ -148,7 +148,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -157,7 +157,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
out.Reset()
|
out.Reset()
|
||||||
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
err = RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(force) error = %v", err)
|
t.Fatalf("RunStage(force) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -168,7 +168,7 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
@@ -184,7 +184,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(force) error = %v", err)
|
t.Fatalf("RunStage(force) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -203,7 +203,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
}
|
}
|
||||||
|
|
||||||
out.Reset()
|
out.Reset()
|
||||||
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
err = Resume(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Resume() error = %v", err)
|
t.Fatalf("Resume() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -214,13 +214,13 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
|
|
||||||
func TestRunStageTrimExecutes(t *testing.T) {
|
func TestRunStageTrimExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "trim"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "trim"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(trim) error = %v", err)
|
t.Fatalf("RunStage(trim) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -243,13 +243,13 @@ func TestRunStageTrimExecutes(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunStageNormalizeExecutes(t *testing.T) {
|
func TestRunStageNormalizeExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "normalize"}, &out)
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "normalize"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage(normalize) error = %v", err)
|
t.Fatalf("RunStage(normalize) error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,13 +15,17 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
||||||
|
|
||||||
@@ -31,17 +35,9 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("run: unexpected positional arguments")
|
return fmt.Errorf("run: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run: %w", err)
|
return fmt.Errorf("run: %w", err)
|
||||||
|
|||||||
@@ -15,13 +15,17 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
|
var campaignPath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
var sessionID string
|
var sessionID string
|
||||||
|
var previousSessionID string
|
||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "path to campaign.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
fs.Var(&selectedArtifacts, "artifacts", "artifact names to execute during analyze (comma-separated or repeatable)")
|
||||||
|
|
||||||
@@ -44,17 +48,9 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
return fmt.Errorf("run-stage: %w", err)
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
cfg, err := loadCommandConfig(ctx, pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
|
|||||||
@@ -555,6 +555,12 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
|
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
if stageRequested("prepare") {
|
||||||
|
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||||
|
if len(requirements) > 0 && strings.TrimSpace(cfg.Session.PreviousSessionID) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
if !stageRequested("archive") {
|
if !stageRequested("archive") {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -569,3 +575,10 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func configuredScriptoriumArtifacts(cfg *config.Config) map[string]config.ScriptoriumArtifactConfig {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return cfg.Pipeline.Scriptorium.Artifacts
|
||||||
|
}
|
||||||
|
|||||||
@@ -199,6 +199,55 @@ func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNeedsObjectStoreForRunPrepareWithPreviousRequirements(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
previousSessionID string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "previous session configured",
|
||||||
|
previousSessionID: "2026-05-10",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "previous session missing",
|
||||||
|
previousSessionID: "",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Scriptorium: &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"previous_recap": {
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Session: &config.SessionConfig{
|
||||||
|
PreviousSessionID: tt.previousSessionID,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := needsObjectStoreForRun(cfg, []stage.Stage{countingStage{name: "prepare", runs: new(int)}})
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("needsObjectStoreForRun() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesArchiveFailsWhenRequiredRecapPromotionMissingForSelectedArtifacts(t *testing.T) {
|
func TestExecuteStagesArchiveFailsWhenRequiredRecapPromotionMissingForSelectedArtifacts(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||||
@@ -945,10 +994,12 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
cfgDir := t.TempDir()
|
cfgDir := t.TempDir()
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
|
||||||
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
mustWriteFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
|
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
@@ -956,8 +1007,27 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
|
|
||||||
return &config.Config{
|
return &config.Config{
|
||||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
@@ -974,6 +1044,7 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + t.TempDir() + `
|
root: ` + t.TempDir() + `
|
||||||
@@ -983,6 +1054,12 @@ analyzer:
|
|||||||
timeout: 20m
|
timeout: 20m
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
|
`
|
||||||
|
campaignYAML := `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
`
|
`
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
@@ -993,6 +1070,7 @@ inputs:
|
|||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`
|
`
|
||||||
mustWriteFile(t, pipelinePath, pipelineYAML)
|
mustWriteFile(t, pipelinePath, pipelineYAML)
|
||||||
|
mustWriteFile(t, campaignPath, campaignYAML)
|
||||||
mustWriteFile(t, sessionPath, sessionYAML)
|
mustWriteFile(t, sessionPath, sessionYAML)
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
cfg, err := config.Load(pipelinePath, sessionPath)
|
||||||
|
|||||||
@@ -9,11 +9,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPlanUsesDiscoveredSessionTemplateWithSessionID(t *testing.T) {
|
func TestPlanUsesDiscoveredSessionTemplateWithSessionIDs(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
sessionTemplate := `session_id: "{{ session_id }}"
|
sessionTemplate := `session_id: "{{ session_id }}"
|
||||||
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
@@ -36,7 +37,11 @@ inputs:
|
|||||||
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session-id", "2026-04-04"}, &out); err != nil {
|
if err := Plan(context.Background(), []string{
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--session-id", "2026-04-04",
|
||||||
|
"--previous-session-id", "2026-03-28",
|
||||||
|
}, &out); err != nil {
|
||||||
t.Fatalf("Plan() error = %v", err)
|
t.Fatalf("Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
|
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
|
||||||
@@ -46,10 +51,10 @@ inputs:
|
|||||||
|
|
||||||
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
@@ -58,12 +63,45 @@ func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
func TestPlanFailsWhenPreviousSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
previous_session_id: 2026-04-26
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
err := Plan(context.Background(), []string{
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--session-id", "2026-05-03",
|
||||||
|
"--previous-session-id", "2026-04-25",
|
||||||
|
}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "previous_session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--campaign", campaignPath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RunStage() error = %v", err)
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,22 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
|||||||
return explicit, nil
|
return explicit, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolved, err := discoverSessionConfigPathWithCandidates(candidates)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if resolved.Path != "" {
|
||||||
|
return resolved.Path, nil
|
||||||
|
}
|
||||||
|
return "", missingSessionConfigError(resolved.Searched, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
type sessionConfigDiscovery struct {
|
||||||
|
Path string
|
||||||
|
Searched []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func discoverSessionConfigPathWithCandidates(candidates []string) (sessionConfigDiscovery, error) {
|
||||||
ordered := make([]string, 0, len(candidates))
|
ordered := make([]string, 0, len(candidates))
|
||||||
for _, raw := range candidates {
|
for _, raw := range candidates {
|
||||||
path := strings.TrimSpace(raw)
|
path := strings.TrimSpace(raw)
|
||||||
@@ -31,19 +47,32 @@ func resolveSessionConfigPathWithCandidates(flagValue string, candidates []strin
|
|||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return filepath.Clean(path), nil
|
return sessionConfigDiscovery{Path: filepath.Clean(path), Searched: ordered}, nil
|
||||||
}
|
}
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("check default session config %q: %w", path, err)
|
return sessionConfigDiscovery{}, fmt.Errorf("check default session config %q: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(ordered) == 0 {
|
return sessionConfigDiscovery{Searched: ordered}, nil
|
||||||
return "", fmt.Errorf("no session config path provided and no default locations configured")
|
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf(
|
|
||||||
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
|
func missingSessionConfigError(searched []string, remoteDetail string) error {
|
||||||
|
ordered := append([]string(nil), searched...)
|
||||||
|
if len(ordered) == 0 {
|
||||||
|
if strings.TrimSpace(remoteDetail) != "" {
|
||||||
|
return fmt.Errorf("no session config path provided and no default locations configured; %s", remoteDetail)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("no session config path provided and no default locations configured")
|
||||||
|
}
|
||||||
|
msg := fmt.Sprintf(
|
||||||
|
"no session config path provided and no default session config found; searched: %s",
|
||||||
strings.Join(ordered, ", "),
|
strings.Join(ordered, ", "),
|
||||||
)
|
)
|
||||||
|
if strings.TrimSpace(remoteDetail) != "" {
|
||||||
|
msg += "; " + strings.TrimSpace(remoteDetail)
|
||||||
|
}
|
||||||
|
msg += "; pass --session to use an explicit path"
|
||||||
|
return fmt.Errorf("%s", msg)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,15 @@ const (
|
|||||||
ArtifactTranscriptFull = "narratio.transcript.full"
|
ArtifactTranscriptFull = "narratio.transcript.full"
|
||||||
ArtifactTranscriptTrimmed = "narratio.transcript.trimmed"
|
ArtifactTranscriptTrimmed = "narratio.transcript.trimmed"
|
||||||
ArtifactBoundsSession = "narratio.bounds.session"
|
ArtifactBoundsSession = "narratio.bounds.session"
|
||||||
|
|
||||||
|
ArtifactProvenancePreviousCacheManifestInput = "manifest.inputs.previous_cache"
|
||||||
|
ArtifactProvenancePreviousCacheFilesystem = "current_session.previous_cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrSessionArtifactNotFound is returned when no readable artifact exists for a known ID.
|
// ErrSessionArtifactNotFound is returned when no readable artifact exists for a known ID.
|
||||||
var ErrSessionArtifactNotFound = errors.New("session artifact not found")
|
var ErrSessionArtifactNotFound = errors.New("session artifact not found")
|
||||||
var configuredArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.[a-z][a-z0-9_]*$`)
|
var configuredArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.[a-z][a-z0-9_]*$`)
|
||||||
|
var previousSessionArtifactSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||||
|
|
||||||
type artifactContentKind string
|
type artifactContentKind string
|
||||||
|
|
||||||
@@ -118,6 +122,21 @@ func IsConfiguredArtifactSource(source string) bool {
|
|||||||
return configuredArtifactSourceRE.MatchString(strings.TrimSpace(source))
|
return configuredArtifactSourceRE.MatchString(strings.TrimSpace(source))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsPreviousSessionArtifactSource returns true when source is narratio.previous_session.artifact.<name>.
|
||||||
|
func IsPreviousSessionArtifactSource(source string) bool {
|
||||||
|
_, ok := PreviousSessionArtifactName(source)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreviousSessionArtifactName extracts <name> from narratio.previous_session.artifact.<name>.
|
||||||
|
func PreviousSessionArtifactName(source string) (string, bool) {
|
||||||
|
matches := previousSessionArtifactSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return matches[1], true
|
||||||
|
}
|
||||||
|
|
||||||
// ResolveSessionArtifact resolves a symbolic source to a readable local session artifact path.
|
// ResolveSessionArtifact resolves a symbolic source to a readable local session artifact path.
|
||||||
// Resolution order is manifest producer outputs first, then canonical session path fallback.
|
// Resolution order is manifest producer outputs first, then canonical session path fallback.
|
||||||
func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source string) (ResolvedSessionArtifact, error) {
|
func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source string) (ResolvedSessionArtifact, error) {
|
||||||
@@ -170,6 +189,9 @@ func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source str
|
|||||||
// configured narratio.artifact.<name> sources through runtime catalog availability.
|
// configured narratio.artifact.<name> sources through runtime catalog availability.
|
||||||
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
|
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
|
||||||
normalized := strings.TrimSpace(source)
|
normalized := strings.TrimSpace(source)
|
||||||
|
if IsPreviousSessionArtifactSource(normalized) {
|
||||||
|
return ResolvePreviousSessionArtifactWithCatalog(paths, m, normalized, catalog)
|
||||||
|
}
|
||||||
if !IsConfiguredArtifactSource(normalized) {
|
if !IsConfiguredArtifactSource(normalized) {
|
||||||
return ResolveSessionArtifact(paths, m, normalized)
|
return ResolveSessionArtifact(paths, m, normalized)
|
||||||
}
|
}
|
||||||
@@ -195,6 +217,72 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolvePreviousSessionArtifactWithCatalog resolves one canonical previous-session source id
|
||||||
|
// to the prepared current-session previous-cache path.
|
||||||
|
func ResolvePreviousSessionArtifactWithCatalog(
|
||||||
|
paths SessionPaths,
|
||||||
|
m *manifest.Manifest,
|
||||||
|
source string,
|
||||||
|
catalog *ArtifactCatalog,
|
||||||
|
) (ResolvedSessionArtifact, error) {
|
||||||
|
artifactName, ok := PreviousSessionArtifactName(source)
|
||||||
|
if !ok {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("unsupported previous-session artifact source %q", source)
|
||||||
|
}
|
||||||
|
if catalog == nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("previous-session artifact source %q requires runtime artifact catalog", source)
|
||||||
|
}
|
||||||
|
|
||||||
|
configuredSourceID := ConfiguredArtifactSourceID(artifactName)
|
||||||
|
entry, ok := catalog.Lookup(configuredSourceID)
|
||||||
|
if !ok {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("unsupported previous-session artifact source %q", source)
|
||||||
|
}
|
||||||
|
candidates := previousSessionCacheCandidatePaths(paths, entry.CanonicalRelPath)
|
||||||
|
if len(candidates) == 0 {
|
||||||
|
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: source}
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestInputPaths := manifestInputPathSet(paths, m)
|
||||||
|
fallback := ""
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
exists, isDir, statErr := pathExists(candidate)
|
||||||
|
if statErr != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", candidate, statErr)
|
||||||
|
}
|
||||||
|
if !exists || isDir {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := validateResolvedContent(candidate, contentText); err != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", source, err)
|
||||||
|
}
|
||||||
|
if _, ok := manifestInputPaths[candidate]; ok {
|
||||||
|
return ResolvedSessionArtifact{
|
||||||
|
ID: source,
|
||||||
|
Path: candidate,
|
||||||
|
ProducerStage: "prepare",
|
||||||
|
OutputKind: "previous_session_artifact",
|
||||||
|
Provenance: ArtifactProvenancePreviousCacheManifestInput,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if fallback == "" {
|
||||||
|
fallback = candidate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if fallback != "" {
|
||||||
|
return ResolvedSessionArtifact{
|
||||||
|
ID: source,
|
||||||
|
Path: fallback,
|
||||||
|
ProducerStage: "prepare",
|
||||||
|
OutputKind: "previous_session_artifact",
|
||||||
|
Provenance: ArtifactProvenancePreviousCacheFilesystem,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: source}
|
||||||
|
}
|
||||||
|
|
||||||
func manifestArtifactCandidates(paths SessionPaths, m *manifest.Manifest, spec artifactSpec) []ResolvedSessionArtifact {
|
func manifestArtifactCandidates(paths SessionPaths, m *manifest.Manifest, spec artifactSpec) []ResolvedSessionArtifact {
|
||||||
if m == nil || len(m.Stages) == 0 || spec.ProducerStage == "" || spec.OutputKind == "" {
|
if m == nil || len(m.Stages) == 0 || spec.ProducerStage == "" || spec.OutputKind == "" {
|
||||||
return nil
|
return nil
|
||||||
@@ -243,6 +331,50 @@ func dedupeResolvedArtifacts(values []ResolvedSessionArtifact) []ResolvedSession
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func previousSessionCacheCandidatePaths(paths SessionPaths, canonicalRelPath string) []string {
|
||||||
|
trimmed := strings.TrimSpace(canonicalRelPath)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
normalized := filepath.ToSlash(filepath.Clean(filepath.FromSlash(trimmed)))
|
||||||
|
if normalized == "." || normalized == "" || normalized == ".." || strings.HasPrefix(normalized, "../") || strings.HasPrefix(normalized, "/") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
relCandidates := []string{normalized}
|
||||||
|
const artifactsPrefix = "artifacts/"
|
||||||
|
if strings.HasPrefix(normalized, artifactsPrefix) && len(normalized) > len(artifactsPrefix) {
|
||||||
|
relCandidates = append(relCandidates, strings.TrimPrefix(normalized, artifactsPrefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]string, 0, len(relCandidates))
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, rel := range relCandidates {
|
||||||
|
abs := filepath.Clean(SessionPreviousArtifactPath(paths, rel))
|
||||||
|
if _, ok := seen[abs]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[abs] = struct{}{}
|
||||||
|
out = append(out, abs)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestInputPathSet(paths SessionPaths, m *manifest.Manifest) map[string]struct{} {
|
||||||
|
if m == nil || len(m.Inputs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]struct{}, len(m.Inputs))
|
||||||
|
for _, in := range m.Inputs {
|
||||||
|
resolved := filepath.Clean(ResolveSessionLocalPathForRead(paths, in.Path))
|
||||||
|
if strings.TrimSpace(resolved) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[resolved] = struct{}{}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func pathExists(path string) (exists bool, isDir bool, err error) {
|
func pathExists(path string) (exists bool, isDir bool, err error) {
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
|||||||
@@ -45,6 +45,58 @@ func TestNormalizeSessionArtifactSource(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreviousSessionArtifactSourceHelpers(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
wantName string
|
||||||
|
wantMatch bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid",
|
||||||
|
source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
wantName: "session_recap",
|
||||||
|
wantMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid with surrounding whitespace",
|
||||||
|
source: " narratio.previous_session.artifact.quest_log ",
|
||||||
|
wantName: "quest_log",
|
||||||
|
wantMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing name",
|
||||||
|
source: "narratio.previous_session.artifact.",
|
||||||
|
wantMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid key characters",
|
||||||
|
source: "narratio.previous_session.artifact.session-recap",
|
||||||
|
wantMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong prefix",
|
||||||
|
source: "narratio.previous.artifact.session_recap",
|
||||||
|
wantMatch: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
name, ok := PreviousSessionArtifactName(tt.source)
|
||||||
|
if ok != tt.wantMatch {
|
||||||
|
t.Fatalf("PreviousSessionArtifactName(%q) ok = %t, want %t", tt.source, ok, tt.wantMatch)
|
||||||
|
}
|
||||||
|
if name != tt.wantName {
|
||||||
|
t.Fatalf("PreviousSessionArtifactName(%q) name = %q, want %q", tt.source, name, tt.wantName)
|
||||||
|
}
|
||||||
|
if got := IsPreviousSessionArtifactSource(tt.source); got != tt.wantMatch {
|
||||||
|
t.Fatalf("IsPreviousSessionArtifactSource(%q) = %t, want %t", tt.source, got, tt.wantMatch)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
@@ -266,3 +318,120 @@ func TestResolveSessionArtifactWithCatalogUnsupportedConfiguredSourceFails(t *te
|
|||||||
t.Fatalf("error = %q, want unsupported artifact source", err.Error())
|
t.Fatalf("error = %q, want unsupported artifact source", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePreviousSessionArtifactWithCatalogPrefersManifestInputRecord(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|
||||||
|
manifestBackedPath := SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
fallbackPath := SessionPreviousArtifactPath(paths, "session_recap.md")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestBackedPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(fallbackPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(manifestBackedPath, []byte("recap from manifest input\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fallbackPath, []byte("recap fallback\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(
|
||||||
|
map[string]ConfiguredArtifactDefinition{
|
||||||
|
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := manifest.New("session", time.Now().UTC())
|
||||||
|
m.Inputs = []manifest.InputRecord{
|
||||||
|
{Kind: "previous_artifact", Path: manifestBackedPath},
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePreviousSessionArtifactWithCatalog(
|
||||||
|
paths,
|
||||||
|
m,
|
||||||
|
"narratio.previous_session.artifact.session_recap",
|
||||||
|
catalog,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePreviousSessionArtifactWithCatalog() error = %v", err)
|
||||||
|
}
|
||||||
|
if resolved.Path != manifestBackedPath {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", resolved.Path, manifestBackedPath)
|
||||||
|
}
|
||||||
|
if resolved.Provenance != ArtifactProvenancePreviousCacheManifestInput {
|
||||||
|
t.Fatalf("provenance = %q, want %q", resolved.Provenance, ArtifactProvenancePreviousCacheManifestInput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePreviousSessionArtifactWithCatalogFallsBackToPreparedCachePath(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|
||||||
|
fallbackPath := SessionPreviousArtifactPath(paths, "session_recap.md")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(fallbackPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fallbackPath, []byte("recap fallback\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(
|
||||||
|
map[string]ConfiguredArtifactDefinition{
|
||||||
|
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePreviousSessionArtifactWithCatalog(
|
||||||
|
paths,
|
||||||
|
nil,
|
||||||
|
"narratio.previous_session.artifact.session_recap",
|
||||||
|
catalog,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePreviousSessionArtifactWithCatalog() error = %v", err)
|
||||||
|
}
|
||||||
|
if resolved.Path != fallbackPath {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", resolved.Path, fallbackPath)
|
||||||
|
}
|
||||||
|
if resolved.Provenance != ArtifactProvenancePreviousCacheFilesystem {
|
||||||
|
t.Fatalf("provenance = %q, want %q", resolved.Provenance, ArtifactProvenancePreviousCacheFilesystem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePreviousSessionArtifactWithCatalogMissingReturnsTypedError(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(
|
||||||
|
map[string]ConfiguredArtifactDefinition{
|
||||||
|
"session_recap": {Enabled: true, OutputPath: "artifacts/session_recap.md"},
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("RegisterConfiguredArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ResolvePreviousSessionArtifactWithCatalog(
|
||||||
|
paths,
|
||||||
|
nil,
|
||||||
|
"narratio.previous_session.artifact.session_recap",
|
||||||
|
catalog,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrSessionArtifactNotFound) {
|
||||||
|
t.Fatalf("errors.Is(err, ErrSessionArtifactNotFound)=false; err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -73,6 +73,8 @@ func (s *LocalStore) ensureLayout(paths SessionPaths) (SessionPaths, error) {
|
|||||||
paths.LogsDir,
|
paths.LogsDir,
|
||||||
paths.CurrentDir,
|
paths.CurrentDir,
|
||||||
paths.RunsDir,
|
paths.RunsDir,
|
||||||
|
paths.PreviousDir,
|
||||||
|
paths.PreviousArtifactsDir,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, dir := range dirs {
|
for _, dir := range dirs {
|
||||||
|
|||||||
@@ -27,10 +27,15 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
|||||||
checkDirExists(t, paths.LogsDir)
|
checkDirExists(t, paths.LogsDir)
|
||||||
checkDirExists(t, paths.CurrentDir)
|
checkDirExists(t, paths.CurrentDir)
|
||||||
checkDirExists(t, paths.RunsDir)
|
checkDirExists(t, paths.RunsDir)
|
||||||
|
checkDirExists(t, paths.PreviousDir)
|
||||||
|
checkDirExists(t, paths.PreviousArtifactsDir)
|
||||||
|
|
||||||
if filepath.Base(paths.ManifestPath) != "manifest.json" {
|
if filepath.Base(paths.ManifestPath) != "manifest.json" {
|
||||||
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
|
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
|
||||||
}
|
}
|
||||||
|
if filepath.Base(paths.PreviousManifestPath) != "manifest.json" {
|
||||||
|
t.Fatalf("PreviousManifestPath = %q, want basename manifest.json", paths.PreviousManifestPath)
|
||||||
|
}
|
||||||
if filepath.Base(paths.LockPath) != ".lock" {
|
if filepath.Base(paths.LockPath) != ".lock" {
|
||||||
t.Fatalf("LockPath = %q, want basename .lock", paths.LockPath)
|
t.Fatalf("LockPath = %q, want basename .lock", paths.LockPath)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,9 @@ type SessionPaths struct {
|
|||||||
LogsDir string
|
LogsDir string
|
||||||
CurrentDir string
|
CurrentDir string
|
||||||
RunsDir string
|
RunsDir string
|
||||||
|
PreviousDir string
|
||||||
|
PreviousManifestPath string
|
||||||
|
PreviousArtifactsDir string
|
||||||
ManifestPath string
|
ManifestPath string
|
||||||
LockPath string
|
LockPath string
|
||||||
}
|
}
|
||||||
@@ -42,6 +45,29 @@ func SessionRunsDirForCampaign(rootDir, campaign, sessionID string) string {
|
|||||||
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathRunsDirSegment)
|
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathRunsDirSegment)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SessionPreviousDirForCampaign returns the canonical previous-session state directory.
|
||||||
|
func SessionPreviousDirForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathPreviousDirSegment)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousManifestPathForCampaign returns the canonical previous-session manifest cache path.
|
||||||
|
func SessionPreviousManifestPathForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(SessionPreviousDirForCampaign(rootDir, campaign, sessionID), config.PathManifestFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousArtifactsDirForCampaign returns the canonical previous-session artifact cache directory.
|
||||||
|
func SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(SessionPreviousDirForCampaign(rootDir, campaign, sessionID), config.PathArtifactsDirSegment)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousArtifactPathForCampaign returns a path under previous/artifacts for one artifact.
|
||||||
|
func SessionPreviousArtifactPathForCampaign(rootDir, campaign, sessionID, artifactRelativePath string) string {
|
||||||
|
return filepath.Join(
|
||||||
|
SessionPreviousArtifactsDirForCampaign(rootDir, campaign, sessionID),
|
||||||
|
filepath.FromSlash(artifactRelativePath),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// SessionRunRootForCampaign returns the canonical run root under runs/{run_id}.
|
// SessionRunRootForCampaign returns the canonical run root under runs/{run_id}.
|
||||||
func SessionRunRootForCampaign(rootDir, campaign, sessionID, runID string) string {
|
func SessionRunRootForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
return filepath.Join(SessionRunsDirForCampaign(rootDir, campaign, sessionID), runID)
|
return filepath.Join(SessionRunsDirForCampaign(rootDir, campaign, sessionID), runID)
|
||||||
@@ -62,6 +88,26 @@ func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
|||||||
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SessionPreviousDir returns the previous-session state directory for already-resolved session paths.
|
||||||
|
func SessionPreviousDir(paths SessionPaths) string {
|
||||||
|
return paths.PreviousDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousManifestPath returns the previous-session manifest path for already-resolved session paths.
|
||||||
|
func SessionPreviousManifestPath(paths SessionPaths) string {
|
||||||
|
return paths.PreviousManifestPath
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousArtifactsDir returns the previous-session artifact directory for already-resolved session paths.
|
||||||
|
func SessionPreviousArtifactsDir(paths SessionPaths) string {
|
||||||
|
return paths.PreviousArtifactsDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionPreviousArtifactPath returns a path under previous/artifacts for already-resolved session paths.
|
||||||
|
func SessionPreviousArtifactPath(paths SessionPaths, artifactRelativePath string) string {
|
||||||
|
return filepath.Join(paths.PreviousArtifactsDir, filepath.FromSlash(artifactRelativePath))
|
||||||
|
}
|
||||||
|
|
||||||
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
||||||
root := SessionWorkDirForCampaign(workspaceRoot, campaign, sessionID)
|
root := SessionWorkDirForCampaign(workspaceRoot, campaign, sessionID)
|
||||||
return buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root)
|
return buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root)
|
||||||
@@ -84,6 +130,9 @@ func buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root string)
|
|||||||
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
|
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
|
||||||
CurrentDir: filepath.Join(root, config.PathCurrentDirSegment),
|
CurrentDir: filepath.Join(root, config.PathCurrentDirSegment),
|
||||||
RunsDir: filepath.Join(root, config.PathRunsDirSegment),
|
RunsDir: filepath.Join(root, config.PathRunsDirSegment),
|
||||||
|
PreviousDir: filepath.Join(root, config.PathPreviousDirSegment),
|
||||||
|
PreviousManifestPath: filepath.Join(root, config.PathPreviousDirSegment, config.PathManifestFile),
|
||||||
|
PreviousArtifactsDir: filepath.Join(root, config.PathPreviousDirSegment, config.PathArtifactsDirSegment),
|
||||||
ManifestPath: filepath.Join(root, config.PathManifestFile),
|
ManifestPath: filepath.Join(root, config.PathManifestFile),
|
||||||
LockPath: filepath.Join(root, config.PathLockFile),
|
LockPath: filepath.Join(root, config.PathLockFile),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,52 @@ func TestSessionRunManifestPathForCampaign(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
previousDir := SessionPreviousDirForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
wantPreviousDir := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous")
|
||||||
|
if previousDir != wantPreviousDir {
|
||||||
|
t.Fatalf("SessionPreviousDirForCampaign() = %q, want %q", previousDir, wantPreviousDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestPath := SessionPreviousManifestPathForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
wantManifestPath := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "manifest.json")
|
||||||
|
if manifestPath != wantManifestPath {
|
||||||
|
t.Fatalf("SessionPreviousManifestPathForCampaign() = %q, want %q", manifestPath, wantManifestPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactsDir := SessionPreviousArtifactsDirForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
wantArtifactsDir := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "artifacts")
|
||||||
|
if artifactsDir != wantArtifactsDir {
|
||||||
|
t.Fatalf("SessionPreviousArtifactsDirForCampaign() = %q, want %q", artifactsDir, wantArtifactsDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactPath := SessionPreviousArtifactPathForCampaign(root, "forsaken", "2026-04-19", "session_recap.md")
|
||||||
|
wantArtifactPath := filepath.Join(root, "work", "forsaken", "2026-04-19", "previous", "artifacts", "session_recap.md")
|
||||||
|
if artifactPath != wantArtifactPath {
|
||||||
|
t.Fatalf("SessionPreviousArtifactPathForCampaign() = %q, want %q", artifactPath, wantArtifactPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionPreviousPathsFromSessionPaths(t *testing.T) {
|
||||||
|
paths := buildSessionPaths("/tmp/workspace", "forsaken", "2026-04-19")
|
||||||
|
if got := SessionPreviousDir(paths); got != paths.PreviousDir {
|
||||||
|
t.Fatalf("SessionPreviousDir() = %q, want %q", got, paths.PreviousDir)
|
||||||
|
}
|
||||||
|
if got := SessionPreviousManifestPath(paths); got != paths.PreviousManifestPath {
|
||||||
|
t.Fatalf("SessionPreviousManifestPath() = %q, want %q", got, paths.PreviousManifestPath)
|
||||||
|
}
|
||||||
|
if got := SessionPreviousArtifactsDir(paths); got != paths.PreviousArtifactsDir {
|
||||||
|
t.Fatalf("SessionPreviousArtifactsDir() = %q, want %q", got, paths.PreviousArtifactsDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := SessionPreviousArtifactPath(paths, "quest_log.json")
|
||||||
|
want := filepath.Join(paths.PreviousArtifactsDir, "quest_log.json")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionPreviousArtifactPath() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSessionSpoolAudioDir(t *testing.T) {
|
func TestSessionSpoolAudioDir(t *testing.T) {
|
||||||
root := "/var/spool/narratio"
|
root := "/var/spool/narratio"
|
||||||
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
|
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
|
||||||
|
|||||||
112
internal/artifacts/previous_requirements.go
Normal file
112
internal/artifacts/previous_requirements.go
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreviousArtifactRequirement describes one previous-session artifact dependency.
|
||||||
|
type PreviousArtifactRequirement struct {
|
||||||
|
Name string
|
||||||
|
Required bool
|
||||||
|
Sources []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CollectPreviousArtifactRequirements scans enabled Scriptorium artifacts and returns
|
||||||
|
// deduplicated previous-session artifact requirements in deterministic order.
|
||||||
|
func CollectPreviousArtifactRequirements(
|
||||||
|
artifactsCfg map[string]config.ScriptoriumArtifactConfig,
|
||||||
|
) []PreviousArtifactRequirement {
|
||||||
|
if len(artifactsCfg) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactNames := sortedScriptoriumArtifactNames(artifactsCfg)
|
||||||
|
byName := map[string]PreviousArtifactRequirement{}
|
||||||
|
|
||||||
|
for _, artifactName := range artifactNames {
|
||||||
|
artifactCfg := artifactsCfg[artifactName]
|
||||||
|
if !artifactCfg.Enabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
inputNames := sortedScriptoriumInputKeys(artifactCfg.Inputs)
|
||||||
|
for _, inputName := range inputNames {
|
||||||
|
inputCfg := artifactCfg.Inputs[inputName]
|
||||||
|
previousName, ok := PreviousSessionArtifactName(inputCfg.Source)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
location := fmt.Sprintf(
|
||||||
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source",
|
||||||
|
artifactName,
|
||||||
|
inputName,
|
||||||
|
)
|
||||||
|
requirement := byName[previousName]
|
||||||
|
requirement.Name = previousName
|
||||||
|
requirement.Required = requirement.Required || inputCfg.Required
|
||||||
|
requirement.Sources = append(requirement.Sources, location)
|
||||||
|
byName[previousName] = requirement
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(byName) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
requirements := make([]PreviousArtifactRequirement, 0, len(byName))
|
||||||
|
for _, requirement := range byName {
|
||||||
|
requirement.Sources = dedupeAndSortStrings(requirement.Sources)
|
||||||
|
requirements = append(requirements, requirement)
|
||||||
|
}
|
||||||
|
sort.Slice(requirements, func(i, j int) bool {
|
||||||
|
return requirements[i].Name < requirements[j].Name
|
||||||
|
})
|
||||||
|
return requirements
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedScriptoriumArtifactNames(artifactsCfg map[string]config.ScriptoriumArtifactConfig) []string {
|
||||||
|
names := make([]string, 0, len(artifactsCfg))
|
||||||
|
for name := range artifactsCfg {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedScriptoriumInputKeys(inputs map[string]config.ScriptoriumInputConfig) []string {
|
||||||
|
if len(inputs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(inputs))
|
||||||
|
for name := range inputs {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func dedupeAndSortStrings(values []string) []string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[trimmed]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[trimmed] = struct{}{}
|
||||||
|
out = append(out, trimmed)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
190
internal/artifacts/previous_requirements_test.go
Normal file
190
internal/artifacts/previous_requirements_test.go
Normal file
@@ -0,0 +1,190 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCollectPreviousArtifactRequirements(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
artifactsCfg map[string]config.ScriptoriumArtifactConfig
|
||||||
|
want []PreviousArtifactRequirement
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no artifacts",
|
||||||
|
artifactsCfg: nil,
|
||||||
|
want: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no previous inputs",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"transcript": {Source: "narratio.transcript.trimmed", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one optional previous input",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"previous_recap": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "session_recap",
|
||||||
|
Required: false,
|
||||||
|
Sources: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one required previous input",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"quest_log": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"previous_quest_log": {Source: "narratio.previous_session.artifact.quest_log", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "quest_log",
|
||||||
|
Required: true,
|
||||||
|
Sources: []string{"pipeline.scriptorium.artifacts.quest_log.inputs.previous_quest_log.source"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate references are deduped",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"a": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"b": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"y": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "session_recap",
|
||||||
|
Required: false,
|
||||||
|
Sources: []string{
|
||||||
|
"pipeline.scriptorium.artifacts.a.inputs.x.source",
|
||||||
|
"pipeline.scriptorium.artifacts.b.inputs.y.source",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required plus optional reference becomes required",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"a": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"b": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"y": {Source: "narratio.previous_session.artifact.session_recap", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "session_recap",
|
||||||
|
Required: true,
|
||||||
|
Sources: []string{
|
||||||
|
"pipeline.scriptorium.artifacts.a.inputs.x.source",
|
||||||
|
"pipeline.scriptorium.artifacts.b.inputs.y.source",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "disabled artifact references are ignored",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"disabled_artifact": {
|
||||||
|
Enabled: false,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"x": {Source: "narratio.previous_session.artifact.session_recap", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"enabled_artifact": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"y": {Source: "narratio.previous_session.artifact.quest_log", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "quest_log",
|
||||||
|
Required: false,
|
||||||
|
Sources: []string{"pipeline.scriptorium.artifacts.enabled_artifact.inputs.y.source"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deterministic ordering",
|
||||||
|
artifactsCfg: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"zz": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"b_input": {Source: "narratio.previous_session.artifact.quest_log", Required: false},
|
||||||
|
"a_input": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"aa": {
|
||||||
|
Enabled: true,
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"c_input": {Source: "narratio.previous_session.artifact.session_recap", Required: false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
want: []PreviousArtifactRequirement{
|
||||||
|
{
|
||||||
|
Name: "quest_log",
|
||||||
|
Required: false,
|
||||||
|
Sources: []string{"pipeline.scriptorium.artifacts.zz.inputs.b_input.source"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "session_recap",
|
||||||
|
Required: false,
|
||||||
|
Sources: []string{
|
||||||
|
"pipeline.scriptorium.artifacts.aa.inputs.c_input.source",
|
||||||
|
"pipeline.scriptorium.artifacts.zz.inputs.a_input.source",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := CollectPreviousArtifactRequirements(tt.artifactsCfg)
|
||||||
|
if !reflect.DeepEqual(got, tt.want) {
|
||||||
|
t.Fatalf("CollectPreviousArtifactRequirements() = %#v, want %#v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,12 @@ func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
|
|||||||
return ensureS3TrailingSlash(key)
|
return ensureS3TrailingSlash(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// S3SessionConfigKey returns the session config key.
|
||||||
|
// Format: {session_prefix}/session.yml
|
||||||
|
func S3SessionConfigKey(sessionPrefix string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), "session.yml")
|
||||||
|
}
|
||||||
|
|
||||||
// S3CurrentManifestKey returns the current manifest pointer key.
|
// S3CurrentManifestKey returns the current manifest pointer key.
|
||||||
// Format: {session_prefix}/current/manifest.json
|
// Format: {session_prefix}/current/manifest.json
|
||||||
func S3CurrentManifestKey(sessionPrefix string) string {
|
func S3CurrentManifestKey(sessionPrefix string) string {
|
||||||
|
|||||||
@@ -17,6 +17,11 @@ func TestS3KeyConstruction(t *testing.T) {
|
|||||||
t.Fatalf("audioPrefix = %q", audioPrefix)
|
t.Fatalf("audioPrefix = %q", audioPrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sessionConfigKey := S3SessionConfigKey(`dnd\campaigns\forsaken\sessions\2026-04-19\`)
|
||||||
|
if sessionConfigKey != "dnd/campaigns/forsaken/sessions/2026-04-19/session.yml" {
|
||||||
|
t.Fatalf("session config key = %q", sessionConfigKey)
|
||||||
|
}
|
||||||
|
|
||||||
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
||||||
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
||||||
if runPrefix != wantRunPrefix {
|
if runPrefix != wantRunPrefix {
|
||||||
|
|||||||
138
internal/config/campaign_config_test.go
Normal file
138
internal/config/campaign_config_test.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCampaignConfigDefaultSearchOrder(t *testing.T) {
|
||||||
|
want := []string{
|
||||||
|
"./campaign.yml",
|
||||||
|
"/usr/local/etc/narratio/campaign.yml",
|
||||||
|
"/etc/narratio/campaign.yml",
|
||||||
|
}
|
||||||
|
if len(DefaultCampaignConfigSearchPaths) != len(want) {
|
||||||
|
t.Fatalf("DefaultCampaignConfigSearchPaths = %#v, want %#v", DefaultCampaignConfigSearchPaths, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if DefaultCampaignConfigSearchPaths[i] != want[i] {
|
||||||
|
t.Fatalf("DefaultCampaignConfigSearchPaths[%d] = %q, want %q", i, DefaultCampaignConfigSearchPaths[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\nunknown: true\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMergeFillsStableInputs(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Session.Campaign != "sample-campaign" {
|
||||||
|
t.Fatalf("session campaign = %q, want campaign config value", cfg.Session.Campaign)
|
||||||
|
}
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./campaign-speakers.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMergeSessionOverridesStableInputs(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./campaign-speakers.yml\n autocorrect_file: ./campaign-autocorrect.yml\n glossary_file: ./campaign-glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ncampaign: sample-campaign\ninputs:\n audio_dir: ./audio\n speakers_file: ./session-speakers.yml\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.SpeakersFile, "./session-speakers.yml", sessionPath, "session_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.AutocorrectFile, "./campaign-autocorrect.yml", campaignPath, "campaign_config")
|
||||||
|
assertResolvedStableInput(t, cfg.StableInputs.GlossaryFile, "./campaign-glossary.yml", campaignPath, "campaign_config")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCampaignSessionMismatchFails(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ncampaign: other-campaign\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "does not match campaign config") {
|
||||||
|
t.Fatalf("error = %q, want campaign mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadMissingCampaignFileFails(t *testing.T) {
|
||||||
|
pipelinePath, _, sessionPath := writeCampaignConfigTestFiles(t,
|
||||||
|
"campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n",
|
||||||
|
"session_id: 2026-05-03\ninputs:\n audio_dir: ./audio\n",
|
||||||
|
)
|
||||||
|
missingCampaignPath := filepath.Join(filepath.Dir(sessionPath), "missing-campaign.yml")
|
||||||
|
|
||||||
|
_, err := LoadWithSessionOptions(pipelinePath, missingCampaignPath, sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected load error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "load campaign config") {
|
||||||
|
t.Fatalf("error = %q, want campaign load context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCampaignConfigTestFiles(t *testing.T, campaignYAML, sessionYAML string) (string, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := "workspace:\n root: " + filepath.ToSlash(filepath.Join(dir, "work")) + "\nwhisperx:\n transcribe_url: https://example.com/transcribe\nanalyzer:\n timeout: 20m\nnotification:\n timeout: 10s\n"
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertResolvedStableInput(t *testing.T, got ResolvedInputFile, wantPath, wantConfigPath, wantSource string) {
|
||||||
|
t.Helper()
|
||||||
|
if got.Path != wantPath || got.ConfigPath != wantConfigPath || got.Source != wantSource {
|
||||||
|
t.Fatalf("resolved input = %#v, want path=%q config_path=%q source=%q", got, wantPath, wantConfigPath, wantSource)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,11 +1,17 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Config is the resolved combined configuration from pipeline.yml and session.yml.
|
// Config is the resolved combined configuration from pipeline.yml,
|
||||||
|
// campaign.yml, and session.yml.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Pipeline *PipelineConfig
|
Pipeline *PipelineConfig
|
||||||
|
Campaign *CampaignConfig
|
||||||
Session *SessionConfig
|
Session *SessionConfig
|
||||||
PipelinePath string
|
PipelinePath string
|
||||||
|
CampaignPath string
|
||||||
SessionPath string
|
SessionPath string
|
||||||
|
|
||||||
|
StableInputs ResolvedStableInputs
|
||||||
|
SessionSource SessionSource
|
||||||
}
|
}
|
||||||
|
|
||||||
// PipelineConfig contains durable pipeline-level settings.
|
// PipelineConfig contains durable pipeline-level settings.
|
||||||
@@ -25,9 +31,23 @@ type PipelineConfig struct {
|
|||||||
Notification NotificationConfig `yaml:"notification"`
|
Notification NotificationConfig `yaml:"notification"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CampaignConfig contains stable campaign-level identity and input defaults.
|
||||||
|
type CampaignConfig struct {
|
||||||
|
Campaign string `yaml:"campaign"`
|
||||||
|
Inputs CampaignInputsConfig `yaml:"inputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CampaignInputsConfig contains stable campaign-level input file references.
|
||||||
|
type CampaignInputsConfig struct {
|
||||||
|
SpeakersFile string `yaml:"speakers_file"`
|
||||||
|
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||||
|
GlossaryFile string `yaml:"glossary_file"`
|
||||||
|
}
|
||||||
|
|
||||||
// SessionConfig contains per-session inputs and metadata.
|
// SessionConfig contains per-session inputs and metadata.
|
||||||
type SessionConfig struct {
|
type SessionConfig struct {
|
||||||
SessionID string `yaml:"session_id"`
|
SessionID string `yaml:"session_id"`
|
||||||
|
PreviousSessionID string `yaml:"previous_session_id"`
|
||||||
Campaign string `yaml:"campaign"`
|
Campaign string `yaml:"campaign"`
|
||||||
Date string `yaml:"date"`
|
Date string `yaml:"date"`
|
||||||
Title string `yaml:"title"`
|
Title string `yaml:"title"`
|
||||||
@@ -75,6 +95,7 @@ type ArchiveConfig struct {
|
|||||||
Enabled *bool `yaml:"enabled"`
|
Enabled *bool `yaml:"enabled"`
|
||||||
UploadRun *bool `yaml:"upload_run"`
|
UploadRun *bool `yaml:"upload_run"`
|
||||||
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
||||||
|
Locks []ArchiveLockRule `yaml:"locks"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchivePromotionRule configures one artifact promotion mapping.
|
// ArchivePromotionRule configures one artifact promotion mapping.
|
||||||
@@ -84,6 +105,13 @@ type ArchivePromotionRule struct {
|
|||||||
Required *bool `yaml:"required"`
|
Required *bool `yaml:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ArchiveLockRule prevents one source-based promotion from overwriting its
|
||||||
|
// top-level archive destination.
|
||||||
|
type ArchiveLockRule struct {
|
||||||
|
Source string `yaml:"source"`
|
||||||
|
Reason string `yaml:"reason"`
|
||||||
|
}
|
||||||
|
|
||||||
// WhisperXConfig configures WhisperX adapter settings.
|
// WhisperXConfig configures WhisperX adapter settings.
|
||||||
type WhisperXConfig struct {
|
type WhisperXConfig struct {
|
||||||
TranscribeURL string `yaml:"transcribe_url"`
|
TranscribeURL string `yaml:"transcribe_url"`
|
||||||
@@ -228,3 +256,29 @@ type SessionInputsConfig struct {
|
|||||||
type SessionAudioS3Input struct {
|
type SessionAudioS3Input struct {
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolvedStableInputs records where stable input file paths came from after
|
||||||
|
// campaign/session merge.
|
||||||
|
type ResolvedStableInputs struct {
|
||||||
|
SpeakersFile ResolvedInputFile
|
||||||
|
AutocorrectFile ResolvedInputFile
|
||||||
|
GlossaryFile ResolvedInputFile
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvedInputFile records one merged config path and its source config file.
|
||||||
|
type ResolvedInputFile struct {
|
||||||
|
Path string
|
||||||
|
ConfigPath string
|
||||||
|
Source string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionSource records where session.yml came from before materialization.
|
||||||
|
type SessionSource struct {
|
||||||
|
Source string
|
||||||
|
LocalPath string
|
||||||
|
S3Bucket string
|
||||||
|
S3Key string
|
||||||
|
S3Size int64
|
||||||
|
S3ETag string
|
||||||
|
SpoolPath string
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,9 @@ package config
|
|||||||
const (
|
const (
|
||||||
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
||||||
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
||||||
|
DefaultCampaignConfigPathLocal = "./campaign.yml"
|
||||||
|
DefaultCampaignConfigPathUsrLocal = "/usr/local/etc/narratio/campaign.yml"
|
||||||
|
DefaultCampaignConfigPathEtc = "/etc/narratio/campaign.yml"
|
||||||
DefaultSessionConfigPathLocal = "./session.yml"
|
DefaultSessionConfigPathLocal = "./session.yml"
|
||||||
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
||||||
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
||||||
@@ -56,6 +59,7 @@ const (
|
|||||||
PathLogsDirSegment = "logs"
|
PathLogsDirSegment = "logs"
|
||||||
PathCurrentDirSegment = "current"
|
PathCurrentDirSegment = "current"
|
||||||
PathRunsDirSegment = "runs"
|
PathRunsDirSegment = "runs"
|
||||||
|
PathPreviousDirSegment = "previous"
|
||||||
PathManifestFile = "manifest.json"
|
PathManifestFile = "manifest.json"
|
||||||
PathLockFile = ".lock"
|
PathLockFile = ".lock"
|
||||||
PathTranscriptMerged = "transcripts/merged.json"
|
PathTranscriptMerged = "transcripts/merged.json"
|
||||||
@@ -87,6 +91,17 @@ var DefaultPipelineConfigSearchPaths = []string{
|
|||||||
DefaultPipelineConfigPathEtc,
|
DefaultPipelineConfigPathEtc,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultCampaignConfigSearchPaths defines the default search order for
|
||||||
|
// campaign.yml when callers do not provide an explicit path.
|
||||||
|
//
|
||||||
|
// Keep this in a variable so future defaults can be extended without changing
|
||||||
|
// call sites.
|
||||||
|
var DefaultCampaignConfigSearchPaths = []string{
|
||||||
|
DefaultCampaignConfigPathLocal,
|
||||||
|
DefaultCampaignConfigPathUsrLocal,
|
||||||
|
DefaultCampaignConfigPathEtc,
|
||||||
|
}
|
||||||
|
|
||||||
// DefaultSessionConfigSearchPaths defines the default search order for
|
// DefaultSessionConfigSearchPaths defines the default search order for
|
||||||
// session.yml when callers do not provide an explicit path.
|
// session.yml when callers do not provide an explicit path.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
@@ -21,6 +22,15 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
|||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
||||||
|
func LoadCampaign(path string) (*CampaignConfig, error) {
|
||||||
|
var cfg CampaignConfig
|
||||||
|
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("load campaign config: %w", err)
|
||||||
|
}
|
||||||
|
return &cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
// LoadSession loads session configuration from a YAML file with strict field checking.
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
||||||
func LoadSession(path string) (*SessionConfig, error) {
|
func LoadSession(path string) (*SessionConfig, error) {
|
||||||
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
||||||
@@ -29,6 +39,7 @@ func LoadSession(path string) (*SessionConfig, error) {
|
|||||||
// SessionLoadOptions configures session template rendering behavior.
|
// SessionLoadOptions configures session template rendering behavior.
|
||||||
type SessionLoadOptions struct {
|
type SessionLoadOptions struct {
|
||||||
SessionID string
|
SessionID string
|
||||||
|
PreviousSessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadSessionWithOptions loads session configuration from a YAML file with
|
// LoadSessionWithOptions loads session configuration from a YAML file with
|
||||||
@@ -38,53 +49,180 @@ func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
||||||
}
|
}
|
||||||
|
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
|
||||||
|
}
|
||||||
|
|
||||||
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
|
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
|
||||||
|
// strict field checking after template rendering.
|
||||||
|
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
|
||||||
|
rendered, err := renderSessionTemplate(string(data), opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load session config: %w", err)
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg SessionConfig
|
var cfg SessionConfig
|
||||||
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
|
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
|
||||||
return nil, fmt.Errorf("load session config: %w", err)
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
||||||
path,
|
label,
|
||||||
strings.TrimSpace(opts.SessionID),
|
strings.TrimSpace(opts.SessionID),
|
||||||
strings.TrimSpace(cfg.SessionID),
|
strings.TrimSpace(cfg.SessionID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
|
||||||
|
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
|
||||||
|
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
||||||
|
label,
|
||||||
|
strings.TrimSpace(opts.PreviousSessionID),
|
||||||
|
strings.TrimSpace(cfg.PreviousSessionID),
|
||||||
|
)
|
||||||
|
}
|
||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load loads and resolves combined pipeline and session configuration.
|
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
||||||
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
// Passing only a session path is supported for package-internal compatibility;
|
||||||
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
|
// in that form campaign.yml is expected next to the session file.
|
||||||
|
func Load(pipelinePath string, paths ...string) (*Config, error) {
|
||||||
|
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadWithSessionOptions loads and resolves combined pipeline and session
|
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
|
||||||
// configuration with session template options.
|
// session configuration with session template options.
|
||||||
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||||
pipelineCfg, err := LoadPipeline(pipelinePath)
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
campaignCfg, err := LoadCampaign(campaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
||||||
|
Source: "session_config",
|
||||||
|
LocalPath: sessionPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve builds final stage-facing configuration from already loaded
|
||||||
|
// pipeline, campaign, and session documents.
|
||||||
|
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
||||||
|
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionSource.Source) == "" {
|
||||||
|
sessionSource.Source = "session_config"
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
||||||
|
sessionSource.LocalPath = sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
return &Config{
|
return &Config{
|
||||||
Pipeline: pipelineCfg,
|
Pipeline: pipelineCfg,
|
||||||
|
Campaign: campaignCfg,
|
||||||
Session: sessionCfg,
|
Session: sessionCfg,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
StableInputs: stableInputs,
|
||||||
|
SessionSource: sessionSource,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
||||||
|
switch len(paths) {
|
||||||
|
case 1:
|
||||||
|
sessionPath = paths[0]
|
||||||
|
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
|
||||||
|
case 2:
|
||||||
|
campaignPath = paths[0]
|
||||||
|
sessionPath = paths[1]
|
||||||
|
default:
|
||||||
|
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
|
||||||
|
}
|
||||||
|
return campaignPath, sessionPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
|
||||||
|
if campaignCfg == nil {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
|
if sessionCfg == nil {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
campaignName := strings.TrimSpace(campaignCfg.Campaign)
|
||||||
|
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
|
||||||
|
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
|
||||||
|
return ResolvedStableInputs{}, fmt.Errorf(
|
||||||
|
"campaign/session config invalid: session campaign %q does not match campaign config %q",
|
||||||
|
sessionCampaign,
|
||||||
|
campaignName,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if sessionCampaign == "" {
|
||||||
|
sessionCfg.Campaign = campaignName
|
||||||
|
}
|
||||||
|
|
||||||
|
stable := ResolvedStableInputs{
|
||||||
|
SpeakersFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.SpeakersFile,
|
||||||
|
sessionCfg.Inputs.SpeakersFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
AutocorrectFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.AutocorrectFile,
|
||||||
|
sessionCfg.Inputs.AutocorrectFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
GlossaryFile: selectStableInput(
|
||||||
|
campaignCfg.Inputs.GlossaryFile,
|
||||||
|
sessionCfg.Inputs.GlossaryFile,
|
||||||
|
campaignPath,
|
||||||
|
sessionPath,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
|
||||||
|
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
|
||||||
|
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
|
||||||
|
return stable, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
|
||||||
|
if strings.TrimSpace(sessionValue) != "" {
|
||||||
|
return ResolvedInputFile{
|
||||||
|
Path: sessionValue,
|
||||||
|
ConfigPath: sessionPath,
|
||||||
|
Source: "session_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ResolvedInputFile{
|
||||||
|
Path: campaignValue,
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func decodeStrictYAML(kind, path string, out any) error {
|
func decodeStrictYAML(kind, path string, out any) error {
|
||||||
f, err := os.Open(path)
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -114,24 +252,36 @@ var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)
|
|||||||
|
|
||||||
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
|
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
|
||||||
sessionID := strings.TrimSpace(opts.SessionID)
|
sessionID := strings.TrimSpace(opts.SessionID)
|
||||||
|
previousSessionID := strings.TrimSpace(opts.PreviousSessionID)
|
||||||
rendered := content
|
rendered := content
|
||||||
if sessionID != "" {
|
if sessionID != "" {
|
||||||
rendered = strings.ReplaceAll(rendered, "{{session_id}}", sessionID)
|
rendered = replaceTemplateVariable(rendered, "session_id", sessionID)
|
||||||
rendered = strings.ReplaceAll(rendered, "{{ session_id }}", sessionID)
|
}
|
||||||
|
if previousSessionID != "" {
|
||||||
|
rendered = replaceTemplateVariable(rendered, "previous_session_id", previousSessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
|
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
|
||||||
if len(unresolved) > 0 {
|
if len(unresolved) > 0 {
|
||||||
|
seenVars := map[string]struct{}{}
|
||||||
vars := make([]string, 0, len(unresolved))
|
vars := make([]string, 0, len(unresolved))
|
||||||
for _, m := range unresolved {
|
for _, m := range unresolved {
|
||||||
if len(m) > 1 {
|
if len(m) > 1 {
|
||||||
vars = append(vars, m[1])
|
name := m[1]
|
||||||
|
if _, ok := seenVars[name]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seenVars[name] = struct{}{}
|
||||||
|
vars = append(vars, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sort.Strings(vars)
|
||||||
if len(vars) > 0 {
|
if len(vars) > 0 {
|
||||||
|
hints := unresolvedTemplateHints(vars)
|
||||||
return "", fmt.Errorf(
|
return "", fmt.Errorf(
|
||||||
"session file template rendering failed: unresolved template variable(s): %s; pass --session-id when using {{ session_id }}",
|
"session file template rendering failed: unresolved template variable(s): %s%s",
|
||||||
strings.Join(vars, ", "),
|
strings.Join(vars, ", "),
|
||||||
|
hints,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
|
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
|
||||||
@@ -140,6 +290,35 @@ func renderSessionTemplate(content string, opts SessionLoadOptions) (string, err
|
|||||||
return rendered, nil
|
return rendered, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func replaceTemplateVariable(content, name, value string) string {
|
||||||
|
rendered := strings.ReplaceAll(content, "{{"+name+"}}", value)
|
||||||
|
rendered = strings.ReplaceAll(rendered, "{{ "+name+" }}", value)
|
||||||
|
return rendered
|
||||||
|
}
|
||||||
|
|
||||||
|
func unresolvedTemplateHints(vars []string) string {
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
flags := make([]string, 0, 2)
|
||||||
|
for _, name := range vars {
|
||||||
|
switch name {
|
||||||
|
case "session_id":
|
||||||
|
if _, ok := seen["--session-id"]; !ok {
|
||||||
|
seen["--session-id"] = struct{}{}
|
||||||
|
flags = append(flags, "--session-id")
|
||||||
|
}
|
||||||
|
case "previous_session_id":
|
||||||
|
if _, ok := seen["--previous-session-id"]; !ok {
|
||||||
|
seen["--previous-session-id"] = struct{}{}
|
||||||
|
flags = append(flags, "--previous-session-id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(flags) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "; pass " + strings.Join(flags, " and ") + " when using those template variable(s)"
|
||||||
|
}
|
||||||
|
|
||||||
func shortName(path, fallback string) string {
|
func shortName(path, fallback string) string {
|
||||||
base := filepath.Base(path)
|
base := filepath.Base(path)
|
||||||
if base == "." || base == string(filepath.Separator) {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
|
|||||||
@@ -187,6 +187,43 @@ inputs:
|
|||||||
`,
|
`,
|
||||||
wantValidate: "session config \"session.yml\" invalid: session.session_id is required",
|
wantValidate: "session config \"session.yml\" invalid: session.session_id is required",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "valid previous_session_id passes",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
previous_session_id: 2026-04-26
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "previous_session_id equal to session_id fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
previous_session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "session config \"session.yml\" invalid: session.previous_session_id must not equal session.session_id",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "missing transcribe_url fails",
|
name: "missing transcribe_url fails",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
@@ -867,6 +904,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
Report: boolPtr(true),
|
Report: boolPtr(true),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Campaign: &CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Session: &SessionConfig{
|
Session: &SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
@@ -926,6 +964,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
|
pipelinePath := filepath.Join(examplesDir, tt.pipelineFile)
|
||||||
|
campaignPath := filepath.Join(examplesDir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
|
sessionPath := filepath.Join(examplesDir, tt.sessionFile)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -935,7 +974,7 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
|
if strings.TrimSpace(tt.sessionOpts.SessionID) == "" {
|
||||||
cfg, err = Load(pipelinePath, sessionPath)
|
cfg, err = Load(pipelinePath, sessionPath)
|
||||||
} else {
|
} else {
|
||||||
cfg, err = LoadWithSessionOptions(pipelinePath, sessionPath, tt.sessionOpts)
|
cfg, err = LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, tt.sessionOpts)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load example config error = %v", err)
|
t.Fatalf("load example config error = %v", err)
|
||||||
@@ -964,14 +1003,34 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline.yml: %v", err)
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
}
|
}
|
||||||
|
campaignYAML := `campaign: ` + campaignNameFromSessionYAML(sessionYAML) + `
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write session.yml: %v", err)
|
t.Fatalf("write session.yml: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, sessionPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func campaignNameFromSessionYAML(sessionYAML string) string {
|
||||||
|
for _, line := range strings.Split(sessionYAML, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
if strings.HasPrefix(trimmed, "campaign:") {
|
||||||
|
return strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "campaign:")), `"'`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "sample-campaign"
|
||||||
|
}
|
||||||
|
|||||||
@@ -116,6 +116,75 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
|
|||||||
output_kind: session_recap
|
output_kind: session_recap
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "canonical previous-session source is accepted",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.polished
|
||||||
|
required: true
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session_recap
|
||||||
|
required: false
|
||||||
|
vars:
|
||||||
|
session_id: true
|
||||||
|
output_kind: session_recap
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "canonical previous-session source missing artifact key fails validation",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.
|
||||||
|
required: false
|
||||||
|
`,
|
||||||
|
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact." must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "canonical previous-session source invalid artifact key fails validation",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session-recap
|
||||||
|
required: false
|
||||||
|
`,
|
||||||
|
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.session-recap" must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "canonical previous-session source unknown artifact fails validation",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.quest_log
|
||||||
|
required: false
|
||||||
|
`,
|
||||||
|
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.quest_log" references unknown artifact "quest_log"`,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "canonical artifact source is accepted",
|
name: "canonical artifact source is accepted",
|
||||||
scriptoriumYAML: `scriptorium:
|
scriptoriumYAML: `scriptorium:
|
||||||
|
|||||||
@@ -55,6 +55,34 @@ inputs:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsRendersPreviousSessionPlaceholder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: "{{ session_id }}"
|
||||||
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{
|
||||||
|
SessionID: "2026-04-04",
|
||||||
|
PreviousSessionID: "2026-03-28",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.PreviousSessionID != "2026-03-28" {
|
||||||
|
t.Fatalf("PreviousSessionID = %q, want 2026-03-28", cfg.PreviousSessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
|
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
@@ -82,6 +110,37 @@ inputs:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsUnresolvedPreviousSessionPlaceholderFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
previous_session_id: "{{ previous_session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "unresolved template variable") {
|
||||||
|
t.Fatalf("error = %q, want unresolved-variable context", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "previous_session_id") {
|
||||||
|
t.Fatalf("error = %q, want previous_session_id variable", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--previous-session-id") {
|
||||||
|
t.Fatalf("error = %q, want previous-session-id guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
|
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
@@ -106,6 +165,34 @@ inputs:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsPreviousSessionMismatchFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
previous_session_id: 2026-04-26
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{
|
||||||
|
SessionID: "2026-05-03",
|
||||||
|
PreviousSessionID: "2026-04-25",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "previous_session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
|
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
@@ -154,3 +241,38 @@ inputs:
|
|||||||
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionBytesWithOptionsUsesSameTemplateAndStrictDecode(t *testing.T) {
|
||||||
|
sessionYAML := []byte(`session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", sessionYAML, SessionLoadOptions{SessionID: "2026-05-03"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionBytesWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SessionID != "2026-05-03" {
|
||||||
|
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\nunknown: true\n"), SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected strict decode error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("error = %q, want strict decode context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionBytesWithOptionsMismatchFails(t *testing.T) {
|
||||||
|
_, err := LoadSessionBytesWithOptions("s3://bucket/session.yml", []byte("session_id: 2026-05-03\n"), SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected mismatch error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -287,6 +287,100 @@ archive:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockValidation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
pipelineYML string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid built in source",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
reason: reviewed transcript
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid configured source",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- reason: no source
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.archive.locks[0].source is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.unknown
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate source rejected",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
- source: " narratio.transcript.trimmed "
|
||||||
|
`,
|
||||||
|
wantErr: "duplicates another archive lock source",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
archive:
|
||||||
|
locks:
|
||||||
|
- source: narratio.transcript.trimmed
|
||||||
|
dest: transcripts/trimmed.json
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||||
|
_, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("Load() error = %v, want strict decode failed", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
archive:
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ func Validate(cfg *Config) error {
|
|||||||
if cfg.Pipeline == nil {
|
if cfg.Pipeline == nil {
|
||||||
return fmt.Errorf("pipeline config is required")
|
return fmt.Errorf("pipeline config is required")
|
||||||
}
|
}
|
||||||
|
if cfg.Campaign == nil {
|
||||||
|
return fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
if cfg.Session == nil {
|
if cfg.Session == nil {
|
||||||
return fmt.Errorf("session config is required")
|
return fmt.Errorf("session config is required")
|
||||||
}
|
}
|
||||||
@@ -24,6 +27,9 @@ func Validate(cfg *Config) error {
|
|||||||
if err := validatePipeline(cfg.Pipeline); err != nil {
|
if err := validatePipeline(cfg.Pipeline); err != nil {
|
||||||
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
||||||
}
|
}
|
||||||
|
if err := validateCampaign(cfg.Campaign); err != nil {
|
||||||
|
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
|
||||||
|
}
|
||||||
if err := validateSession(cfg.Session); err != nil {
|
if err := validateSession(cfg.Session); err != nil {
|
||||||
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
||||||
}
|
}
|
||||||
@@ -34,6 +40,16 @@ func Validate(cfg *Config) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCampaign(cfg *CampaignConfig) error {
|
||||||
|
if cfg == nil {
|
||||||
|
return fmt.Errorf("campaign config is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.Campaign) == "" {
|
||||||
|
return fmt.Errorf("campaign.campaign is required")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validatePipeline(cfg *PipelineConfig) error {
|
func validatePipeline(cfg *PipelineConfig) error {
|
||||||
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
||||||
return fmt.Errorf("pipeline.workspace.root is required")
|
return fmt.Errorf("pipeline.workspace.root is required")
|
||||||
@@ -136,6 +152,23 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
seenDest[normalizedDest] = struct{}{}
|
seenDest[normalizedDest] = struct{}{}
|
||||||
}
|
}
|
||||||
|
seenLocks := map[string]struct{}{}
|
||||||
|
for i, item := range cfg.Locks {
|
||||||
|
prefix := fmt.Sprintf("pipeline.archive.locks[%d]", i)
|
||||||
|
source := strings.TrimSpace(item.Source)
|
||||||
|
if source == "" {
|
||||||
|
return fmt.Errorf("%s.source is required", prefix)
|
||||||
|
}
|
||||||
|
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
|
||||||
|
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
|
}
|
||||||
|
if _, ok := seenLocks[source]; ok {
|
||||||
|
return fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
|
||||||
|
}
|
||||||
|
seenLocks[source] = struct{}{}
|
||||||
|
cfg.Locks[i].Source = source
|
||||||
|
cfg.Locks[i].Reason = strings.TrimSpace(item.Reason)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -487,8 +520,14 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateSession(cfg *SessionConfig) error {
|
func validateSession(cfg *SessionConfig) error {
|
||||||
if strings.TrimSpace(cfg.SessionID) == "" {
|
if err := validateSessionIdentifier("session.session_id", cfg.SessionID, true); err != nil {
|
||||||
return fmt.Errorf("session.session_id is required")
|
return err
|
||||||
|
}
|
||||||
|
if err := validateSessionIdentifier("session.previous_session_id", cfg.PreviousSessionID, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.PreviousSessionID) != "" && strings.TrimSpace(cfg.PreviousSessionID) == strings.TrimSpace(cfg.SessionID) {
|
||||||
|
return fmt.Errorf("session.previous_session_id must not equal session.session_id")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Campaign) == "" {
|
if strings.TrimSpace(cfg.Campaign) == "" {
|
||||||
return fmt.Errorf("session.campaign is required")
|
return fmt.Errorf("session.campaign is required")
|
||||||
@@ -525,6 +564,16 @@ func validateSession(cfg *SessionConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateSessionIdentifier(fieldName, value string, required bool) error {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
if required {
|
||||||
|
return fmt.Errorf("%s is required", fieldName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
|
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
|
||||||
if pipeline == nil || session == nil {
|
if pipeline == nil || session == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -563,12 +612,38 @@ var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
|||||||
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
||||||
var narratioArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
|
var narratioArtifactSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||||
|
var narratioPreviousSessionArtifactSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||||
|
|
||||||
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
|
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
|
||||||
if isStaticSupportedScriptoriumInputSource(source) {
|
trimmedSource := strings.TrimSpace(source)
|
||||||
|
if isStaticSupportedScriptoriumInputSource(trimmedSource) {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
matches := narratioArtifactSourceRE.FindStringSubmatch(source)
|
|
||||||
|
if strings.HasPrefix(trimmedSource, "narratio.previous_session.artifact") {
|
||||||
|
matches := narratioPreviousSessionArtifactSourceRE.FindStringSubmatch(trimmedSource)
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$",
|
||||||
|
artifactName,
|
||||||
|
inputName,
|
||||||
|
source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
referenced := matches[1]
|
||||||
|
if _, ok := configuredArtifacts[referenced]; !ok {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q",
|
||||||
|
artifactName,
|
||||||
|
inputName,
|
||||||
|
source,
|
||||||
|
referenced,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
matches := narratioArtifactSourceRE.FindStringSubmatch(trimmedSource)
|
||||||
if len(matches) != 2 {
|
if len(matches) != 2 {
|
||||||
return "", fmt.Errorf(
|
return "", fmt.Errorf(
|
||||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||||
@@ -591,7 +666,7 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
|
|||||||
}
|
}
|
||||||
|
|
||||||
func isStaticSupportedScriptoriumInputSource(source string) bool {
|
func isStaticSupportedScriptoriumInputSource(source string) bool {
|
||||||
switch strings.TrimSpace(source) {
|
switch source {
|
||||||
case "previous_session_artifact":
|
case "previous_session_artifact":
|
||||||
return true
|
return true
|
||||||
case "narratio.transcript.merged":
|
case "narratio.transcript.merged":
|
||||||
|
|||||||
@@ -631,6 +631,23 @@ func resolveScriptoriumInput(
|
|||||||
runtimeCatalog *artifacts.ArtifactCatalog,
|
runtimeCatalog *artifacts.ArtifactCatalog,
|
||||||
) (string, bool, *artifacts.ResolvedSessionArtifact, error) {
|
) (string, bool, *artifacts.ResolvedSessionArtifact, error) {
|
||||||
source := strings.TrimSpace(inputCfg.Source)
|
source := strings.TrimSpace(inputCfg.Source)
|
||||||
|
if artifacts.IsPreviousSessionArtifactSource(source) {
|
||||||
|
resolved, err := artifacts.ResolvePreviousSessionArtifactWithCatalog(paths, m, source, runtimeCatalog)
|
||||||
|
if err == nil {
|
||||||
|
copy := resolved
|
||||||
|
return resolved.Path, true, ©, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
|
if inputCfg.Required {
|
||||||
|
return "", false, nil, fmt.Errorf(
|
||||||
|
"required previous-session input source %q is unavailable; run narratio run-stage --force prepare",
|
||||||
|
source,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return "", false, nil, nil
|
||||||
|
}
|
||||||
|
return "", false, nil, err
|
||||||
|
}
|
||||||
switch source {
|
switch source {
|
||||||
case "previous_session_artifact":
|
case "previous_session_artifact":
|
||||||
if strings.TrimSpace(inputCfg.Path) == "" {
|
if strings.TrimSpace(inputCfg.Path) == "" {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -717,6 +718,146 @@ func TestAnalyzeOmitsOptionalMissingConfiguredArtifactInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequiredPreviousSessionArtifactInputGuidesPrepareForce(t *testing.T) {
|
||||||
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
|
|
||||||
|
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "run narratio run-stage --force prepare") {
|
||||||
|
t.Fatalf("error = %q, want guidance to run force prepare", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeResolvesCanonicalPreviousSessionArtifactFromManifestInput(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
writeAnalyzeFile(t, previousPath, "previous recap\n")
|
||||||
|
m.Inputs = append(m.Inputs, manifest.InputRecord{
|
||||||
|
Kind: "previous_artifact",
|
||||||
|
Path: previousPath,
|
||||||
|
})
|
||||||
|
|
||||||
|
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if got := fake.RunRequests[0].InputPaths["previous_recap"]; got != previousPath {
|
||||||
|
t.Fatalf("previous_recap input = %q, want %q", got, previousPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeOmitsOptionalMissingCanonicalPreviousSessionArtifact(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: false,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if _, exists := fake.RunRequests[0].InputPaths["previous_recap"]; exists {
|
||||||
|
t.Fatalf("optional canonical previous_recap should be omitted when unavailable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRenderDebugWithCanonicalPreviousSessionInput(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
|
|
||||||
|
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
writeAnalyzeFile(t, previousPath, "previous recap\n")
|
||||||
|
m.Inputs = append(m.Inputs, manifest.InputRecord{
|
||||||
|
Kind: "previous_artifact",
|
||||||
|
Path: previousPath,
|
||||||
|
})
|
||||||
|
|
||||||
|
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RenderRequests) != 1 || len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("render/run requests = %d/%d, want 1/1", len(fake.RenderRequests), len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if got := fake.RenderRequests[0].InputPaths["previous_recap"]; got != previousPath {
|
||||||
|
t.Fatalf("render previous_recap input = %q, want %q", got, previousPath)
|
||||||
|
}
|
||||||
|
if got := fake.RunRequests[0].InputPaths["previous_recap"]; got != previousPath {
|
||||||
|
t.Fatalf("run previous_recap input = %q, want %q", got, previousPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeDoesNotCallObjectStoreForCanonicalPreviousSessionInput(t *testing.T) {
|
||||||
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
previousPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
writeAnalyzeFile(t, previousPath, "previous recap\n")
|
||||||
|
m.Inputs = append(m.Inputs, manifest.InputRecord{
|
||||||
|
Kind: "previous_artifact",
|
||||||
|
Path: previousPath,
|
||||||
|
})
|
||||||
|
|
||||||
|
sessionRecap := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
sessionRecap.Inputs["previous_recap"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = sessionRecap
|
||||||
|
|
||||||
|
tracker := &analyzeObjectStoreTracker{}
|
||||||
|
env.ObjectStore = tracker
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if tracker.called {
|
||||||
|
t.Fatal("analyze should not call object store for previous-session input resolution")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
|
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := sessionPathsForEnv(env, m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
@@ -1145,3 +1286,27 @@ func mustArtifactEntryList(t *testing.T, metadata map[string]any, key string) []
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type analyzeObjectStoreTracker struct {
|
||||||
|
called bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *analyzeObjectStoreTracker) List(context.Context, string) ([]storage.ObjectInfo, error) {
|
||||||
|
s.called = true
|
||||||
|
return nil, errors.New("unexpected object store list call")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *analyzeObjectStoreTracker) Download(context.Context, string, string) error {
|
||||||
|
s.called = true
|
||||||
|
return errors.New("unexpected object store download call")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *analyzeObjectStoreTracker) Upload(context.Context, string, string, storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
|
s.called = true
|
||||||
|
return storage.ObjectInfo{}, errors.New("unexpected object store upload call")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *analyzeObjectStoreTracker) Exists(context.Context, string) (bool, error) {
|
||||||
|
s.called = true
|
||||||
|
return false, errors.New("unexpected object store exists call")
|
||||||
|
}
|
||||||
|
|||||||
@@ -118,11 +118,22 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("archive: collect run files: %w", err)
|
return nil, fmt.Errorf("archive: collect run files: %w", err)
|
||||||
}
|
}
|
||||||
sessionPaths := archiveSessionPaths(env, m)
|
sessionPaths := archiveSessionPaths(env, m)
|
||||||
|
previousFiles, err := collectArchivePreviousFiles(sessionPaths.PreviousDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: collect previous files: %w", err)
|
||||||
|
}
|
||||||
runtimeCatalog, err := buildArchiveRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
runtimeCatalog, err := buildArchiveRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
||||||
}
|
}
|
||||||
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
|
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
|
||||||
|
sessionPaths,
|
||||||
|
m,
|
||||||
|
runtimeCatalog,
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts,
|
||||||
|
env.Config.Pipeline.Archive.Locks,
|
||||||
|
sessionPrefix,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
||||||
}
|
}
|
||||||
@@ -144,6 +155,15 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
promotedUploaded = append(promotedUploaded, promotion.Dest)
|
promotedUploaded = append(promotedUploaded, promotion.Dest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
previousUploaded := make([]string, 0, len(previousFiles))
|
||||||
|
for _, file := range previousFiles {
|
||||||
|
key := artifacts.S3PromotedArtifactKey(sessionPrefix, file.RelativePath)
|
||||||
|
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: upload previous file %q to %q: %w", file.RelativePath, key, err)
|
||||||
|
}
|
||||||
|
previousUploaded = append(previousUploaded, file.RelativePath)
|
||||||
|
}
|
||||||
|
|
||||||
currentManifestKey, currentRunPointerKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
currentManifestKey, currentRunPointerKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
||||||
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
|
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
|
||||||
bucket,
|
bucket,
|
||||||
@@ -151,7 +171,9 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
sessionPrefix,
|
sessionPrefix,
|
||||||
runUploaded,
|
runUploaded,
|
||||||
promotedUploaded,
|
promotedUploaded,
|
||||||
|
previousUploaded,
|
||||||
skippedOptional,
|
skippedOptional,
|
||||||
|
lockedPromotions,
|
||||||
currentManifestKey,
|
currentManifestKey,
|
||||||
))
|
))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,7 +209,11 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
"run_uploaded_paths": runUploaded,
|
"run_uploaded_paths": runUploaded,
|
||||||
"promoted_files_uploaded": len(promotedUploaded),
|
"promoted_files_uploaded": len(promotedUploaded),
|
||||||
"promoted_paths": promotedUploaded,
|
"promoted_paths": promotedUploaded,
|
||||||
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
|
"previous_uploaded_paths": previousUploaded,
|
||||||
"skipped_optional_promotions": skippedOptional,
|
"skipped_optional_promotions": skippedOptional,
|
||||||
|
"locked_promotion_count": len(lockedPromotions),
|
||||||
|
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": currentRunPointerKey,
|
"current_run_id_key": currentRunPointerKey,
|
||||||
"current_pointer_written": true,
|
"current_pointer_written": true,
|
||||||
@@ -204,6 +230,16 @@ type archivePromotion struct {
|
|||||||
Provenance string
|
Provenance string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type archiveLockedPromotion struct {
|
||||||
|
Source string
|
||||||
|
Dest string
|
||||||
|
RemoteKey string
|
||||||
|
Reason string
|
||||||
|
Required bool
|
||||||
|
LocalPath string
|
||||||
|
Provenance string
|
||||||
|
}
|
||||||
|
|
||||||
func archiveDisabled(env *Env) bool {
|
func archiveDisabled(env *Env) bool {
|
||||||
cfg := env.Config.Pipeline.Archive
|
cfg := env.Config.Pipeline.Archive
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
@@ -303,26 +339,53 @@ func resolveArchivePromotions(
|
|||||||
m *manifest.Manifest,
|
m *manifest.Manifest,
|
||||||
catalog *artifacts.ArtifactCatalog,
|
catalog *artifacts.ArtifactCatalog,
|
||||||
rules []config.ArchivePromotionRule,
|
rules []config.ArchivePromotionRule,
|
||||||
) ([]archivePromotion, []string, error) {
|
locks []config.ArchiveLockRule,
|
||||||
|
sessionPrefix string,
|
||||||
|
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
|
||||||
out := make([]archivePromotion, 0, len(rules))
|
out := make([]archivePromotion, 0, len(rules))
|
||||||
skippedOptional := make([]string, 0)
|
skippedOptional := make([]string, 0)
|
||||||
|
lockedPromotions := make([]archiveLockedPromotion, 0)
|
||||||
|
lockSet := archiveLockSet(locks)
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
required := rule.Required == nil || *rule.Required
|
required := rule.Required == nil || *rule.Required
|
||||||
dest, err := resolveArchivePromotionDest(rule, catalog)
|
dest, err := resolveArchivePromotionDest(rule, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("source %q: %w", source, err)
|
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||||
}
|
}
|
||||||
|
lock, locked := lockSet[source]
|
||||||
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if locked {
|
||||||
|
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||||
|
Source: source,
|
||||||
|
Dest: dest,
|
||||||
|
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||||
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
|
Required: required,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
||||||
skippedOptional = append(skippedOptional, dest)
|
skippedOptional = append(skippedOptional, dest)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||||
}
|
}
|
||||||
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||||
|
}
|
||||||
|
if locked {
|
||||||
|
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||||
|
Source: source,
|
||||||
|
Dest: dest,
|
||||||
|
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||||
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
|
Required: required,
|
||||||
|
LocalPath: resolved.Path,
|
||||||
|
Provenance: resolved.Provenance,
|
||||||
|
})
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, archivePromotion{
|
out = append(out, archivePromotion{
|
||||||
Source: source,
|
Source: source,
|
||||||
@@ -332,7 +395,21 @@ func resolveArchivePromotions(
|
|||||||
Provenance: resolved.Provenance,
|
Provenance: resolved.Provenance,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out, skippedOptional, nil
|
return out, skippedOptional, lockedPromotions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||||
|
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||||
|
for _, lock := range locks {
|
||||||
|
source := strings.TrimSpace(lock.Source)
|
||||||
|
if source == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lock.Source = source
|
||||||
|
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||||
|
out[source] = lock
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
||||||
@@ -497,6 +574,51 @@ func collectArchiveRunFiles(runRoot, manifestPath string) ([]archiveUploadFile,
|
|||||||
return files, nil
|
return files, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func collectArchivePreviousFiles(previousDir string) ([]archiveUploadFile, error) {
|
||||||
|
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
|
||||||
|
if previousDir == "" {
|
||||||
|
return nil, fmt.Errorf("previous directory is required")
|
||||||
|
}
|
||||||
|
info, err := os.Stat(previousDir)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("stat %q: %w", previousDir, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return nil, fmt.Errorf("previous path %q is not a directory", previousDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
files := make([]archiveUploadFile, 0, 16)
|
||||||
|
err = filepath.WalkDir(previousDir, func(path string, d fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(previousDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("relative path from %q to %q: %w", previousDir, path, err)
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
files = append(files, archiveUploadFile{
|
||||||
|
RelativePath: filepath.ToSlash(filepath.Join(config.PathPreviousDirSegment, rel)),
|
||||||
|
LocalPath: path,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("walk %q: %w", previousDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(files, func(i, j int) bool {
|
||||||
|
return files[i].RelativePath < files[j].RelativePath
|
||||||
|
})
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
func resolveArchiveRunManifestSource(runRoot string) (string, error) {
|
func resolveArchiveRunManifestSource(runRoot string) (string, error) {
|
||||||
path := filepath.Join(filepath.Clean(runRoot), "manifest.json")
|
path := filepath.Join(filepath.Clean(runRoot), "manifest.json")
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
@@ -600,7 +722,9 @@ func archiveMetadataPreview(
|
|||||||
bucket, runPrefix, sessionPrefix string,
|
bucket, runPrefix, sessionPrefix string,
|
||||||
runUploaded []string,
|
runUploaded []string,
|
||||||
promotedUploaded []string,
|
promotedUploaded []string,
|
||||||
|
previousUploaded []string,
|
||||||
skippedOptional []string,
|
skippedOptional []string,
|
||||||
|
lockedPromotions []archiveLockedPromotion,
|
||||||
currentManifestKey string,
|
currentManifestKey string,
|
||||||
) map[string]any {
|
) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
@@ -612,10 +736,30 @@ func archiveMetadataPreview(
|
|||||||
"run_uploaded_paths": append([]string(nil), runUploaded...),
|
"run_uploaded_paths": append([]string(nil), runUploaded...),
|
||||||
"promoted_files_uploaded": len(promotedUploaded),
|
"promoted_files_uploaded": len(promotedUploaded),
|
||||||
"promoted_paths": append([]string(nil), promotedUploaded...),
|
"promoted_paths": append([]string(nil), promotedUploaded...),
|
||||||
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
|
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
||||||
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
||||||
|
"locked_promotion_count": len(lockedPromotions),
|
||||||
|
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||||
"current_pointer_written": false,
|
"current_pointer_written": false,
|
||||||
"audio_upload_skipped": true,
|
"audio_upload_skipped": true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
|
||||||
|
out := make([]map[string]any, 0, len(locked))
|
||||||
|
for _, item := range locked {
|
||||||
|
out = append(out, map[string]any{
|
||||||
|
"source": item.Source,
|
||||||
|
"dest": item.Dest,
|
||||||
|
"remote_key": item.RemoteKey,
|
||||||
|
"reason": item.Reason,
|
||||||
|
"required": item.Required,
|
||||||
|
"local_path": item.LocalPath,
|
||||||
|
"provenance": item.Provenance,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package stage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -130,6 +131,50 @@ func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
|
|||||||
if result.Metadata["promoted_files_uploaded"] != 2 {
|
if result.Metadata["promoted_files_uploaded"] != 2 {
|
||||||
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
|
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
|
||||||
}
|
}
|
||||||
|
if result.Metadata["previous_files_uploaded"] != 0 {
|
||||||
|
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveUploadsPreviousCacheWhenPresent(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(
|
||||||
|
env.Config.Pipeline.Workspace.Root,
|
||||||
|
env.Config.Session.Campaign,
|
||||||
|
env.Config.Session.SessionID,
|
||||||
|
)
|
||||||
|
writeStageTestFile(t, filepath.Join(sessionRoot, "previous", "manifest.json"), "{\"session_id\":\"2026-04-12\"}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md"), "# previous recap\n")
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
previousManifestKey := m.S3SessionPrefix + "previous/manifest.json"
|
||||||
|
previousRecapKey := m.S3SessionPrefix + "previous/artifacts/session_recap.md"
|
||||||
|
if _, ok := fake.Objects[previousManifestKey]; !ok {
|
||||||
|
t.Fatalf("missing archived previous manifest key %q", previousManifestKey)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[previousRecapKey]; !ok {
|
||||||
|
t.Fatalf("missing archived previous artifact key %q", previousRecapKey)
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_files_uploaded"] != 2 {
|
||||||
|
t.Fatalf("metadata previous_files_uploaded = %#v, want 2", result.Metadata["previous_files_uploaded"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveToleratesMissingPreviousCache(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_files_uploaded"] != 0 {
|
||||||
|
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
||||||
@@ -171,6 +216,128 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.trimmed", Reason: "human reviewed"},
|
||||||
|
}
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||||
|
if _, ok := fake.Objects[trimmedKey]; ok {
|
||||||
|
t.Fatalf("locked promotion key %q should not be uploaded", trimmedKey)
|
||||||
|
}
|
||||||
|
recapKey := m.S3SessionPrefix + "artifacts/session_recap.md"
|
||||||
|
if _, ok := fake.Objects[recapKey]; !ok {
|
||||||
|
t.Fatalf("unlocked promotion key %q should be uploaded", recapKey)
|
||||||
|
}
|
||||||
|
runTrimmedKey := m.S3RunPrefix + "trim/outputs/transcripts/trimmed.json"
|
||||||
|
if _, ok := fake.Objects[runTrimmedKey]; !ok {
|
||||||
|
t.Fatalf("run-local locked source output %q should still be uploaded", runTrimmedKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
currentRunIDKey := m.S3SessionPrefix + "current/run_id.txt"
|
||||||
|
if len(fake.Uploads) == 0 || fake.Uploads[len(fake.Uploads)-1].Key != currentRunIDKey {
|
||||||
|
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Metadata["promoted_files_uploaded"] != 1 {
|
||||||
|
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
|
||||||
|
}
|
||||||
|
if result.Metadata["locked_promotion_count"] != 1 {
|
||||||
|
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
|
||||||
|
}
|
||||||
|
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||||
|
if len(locked) != 1 {
|
||||||
|
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||||
|
}
|
||||||
|
if locked[0]["source"] != "narratio.transcript.trimmed" ||
|
||||||
|
locked[0]["dest"] != "transcripts/trimmed.json" ||
|
||||||
|
locked[0]["remote_key"] != trimmedKey ||
|
||||||
|
locked[0]["reason"] != "human reviewed" ||
|
||||||
|
locked[0]["required"] != true ||
|
||||||
|
locked[0]["local_path"] == "" ||
|
||||||
|
locked[0]["provenance"] == "" {
|
||||||
|
t.Fatalf("locked promotion metadata = %#v", locked[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
currentManifestKey := m.S3SessionPrefix + "current/manifest.json"
|
||||||
|
var current map[string]any
|
||||||
|
if err := json.Unmarshal(fake.Objects[currentManifestKey].Data, ¤t); err != nil {
|
||||||
|
t.Fatalf("unmarshal current manifest: %v", err)
|
||||||
|
}
|
||||||
|
stages := current["stages"].(map[string]any)
|
||||||
|
archive := stages["archive"].(map[string]any)
|
||||||
|
meta := archive["metadata"].(map[string]any)
|
||||||
|
if meta["locked_promotion_count"] != float64(1) {
|
||||||
|
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
|
||||||
|
}
|
||||||
|
items := meta["locked_promotions"].([]any)
|
||||||
|
if len(items) != 1 {
|
||||||
|
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{Source: "narratio.transcript.merged", Dest: "transcripts/merged.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.merged", Reason: "manual merge is locked"},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
mergedKey := m.S3SessionPrefix + "transcripts/merged.json"
|
||||||
|
if _, ok := fake.Objects[mergedKey]; ok {
|
||||||
|
t.Fatalf("locked missing promotion key %q should not be uploaded", mergedKey)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
||||||
|
t.Fatalf("current run pointer should be written for locked missing promotion")
|
||||||
|
}
|
||||||
|
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
||||||
|
if len(locked) != 1 {
|
||||||
|
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
||||||
|
}
|
||||||
|
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
|
||||||
|
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
||||||
|
{Source: "narratio.transcript.trimmed", Reason: "already published"},
|
||||||
|
}
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: trimmedKey, Data: []byte("previously published\n")})
|
||||||
|
|
||||||
|
if _, err := (archiveStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := string(fake.Objects[trimmedKey].Data)
|
||||||
|
if got != "previously published\n" {
|
||||||
|
t.Fatalf("locked promotion object contents = %q, want existing object preserved", got)
|
||||||
|
}
|
||||||
|
for _, upload := range fake.Uploads {
|
||||||
|
if upload.Key == trimmedKey {
|
||||||
|
t.Fatalf("locked promotion key %q was uploaded", trimmedKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
|||||||
@@ -30,8 +30,10 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
store := artifacts.NewLocalStore(root)
|
store := artifacts.NewLocalStore(root)
|
||||||
cfgDir := t.TempDir()
|
cfgDir := t.TempDir()
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
|
writeStageTestFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||||
|
writeStageTestFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
|
writeStageTestFile(t, pipelinePath, "workspace:\n root: "+root+"\n")
|
||||||
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeStageTestFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeStageTestFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
@@ -48,7 +50,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
env := &Env{
|
env := &Env{
|
||||||
Config: &config.Config{
|
Config: &config.Config{
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Pipeline: &config.PipelineConfig{
|
Pipeline: &config.PipelineConfig{
|
||||||
Workspace: config.WorkspaceConfig{Root: root},
|
Workspace: config.WorkspaceConfig{Root: root},
|
||||||
Storage: config.StorageConfig{
|
Storage: config.StorageConfig{
|
||||||
@@ -62,14 +66,28 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
|
||||||
AutocorrectFile: "./autocorrect.yml",
|
|
||||||
GlossaryFile: "./glossary.yml",
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ func (prepareStage) Name() string { return "prepare" }
|
|||||||
func (prepareStage) Declares() IODecl {
|
func (prepareStage) Declares() IODecl {
|
||||||
return IODecl{
|
return IODecl{
|
||||||
Inputs: []artifacts.Ref{
|
Inputs: []artifacts.Ref{
|
||||||
|
{Kind: "config", Category: "inputs", RelativePath: "campaign.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "session.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "pipeline.resolved.yml"},
|
||||||
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
{Kind: "config", Category: "inputs", RelativePath: "speakers.yml"},
|
||||||
@@ -59,21 +60,30 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
campaignSrc := env.Config.CampaignPath
|
||||||
|
if err := requireFile(campaignSrc, "campaign.yml"); err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
sessionSrc := env.Config.SessionPath
|
sessionSrc := env.Config.SessionPath
|
||||||
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
if err := requireFile(sessionSrc, "session.yml"); err != nil {
|
||||||
return nil, fmt.Errorf("prepare: %w", err)
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
}
|
}
|
||||||
sessionDir := filepath.Dir(sessionSrc)
|
sessionDir := filepath.Dir(sessionSrc)
|
||||||
|
|
||||||
speakersSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.SpeakersFile)
|
speakersInput := stableInputSource(env.Config.StableInputs.SpeakersFile, env.Config.Session.Inputs.SpeakersFile, sessionSrc)
|
||||||
|
autocorrectInput := stableInputSource(env.Config.StableInputs.AutocorrectFile, env.Config.Session.Inputs.AutocorrectFile, sessionSrc)
|
||||||
|
glossaryInput := stableInputSource(env.Config.StableInputs.GlossaryFile, env.Config.Session.Inputs.GlossaryFile, sessionSrc)
|
||||||
|
|
||||||
|
speakersSrc, err := resolveConfigRelativePath(speakersInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
return nil, fmt.Errorf("prepare: speakers path: %w", err)
|
||||||
}
|
}
|
||||||
autocorrectSrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.AutocorrectFile)
|
autocorrectSrc, err := resolveConfigRelativePath(autocorrectInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
return nil, fmt.Errorf("prepare: autocorrect path: %w", err)
|
||||||
}
|
}
|
||||||
glossarySrc, err := resolvePath(sessionDir, env.Config.Session.Inputs.GlossaryFile)
|
glossarySrc, err := resolveConfigRelativePath(glossaryInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
return nil, fmt.Errorf("prepare: glossary path: %w", err)
|
||||||
}
|
}
|
||||||
@@ -96,17 +106,44 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
|
inputs := make([]manifest.InputRecord, 0, 6+len(resolvedLocalAudio))
|
||||||
registerInput := func(kind, path, checksum string) {
|
registerInput := func(kind, path, checksum string) {
|
||||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
||||||
}
|
}
|
||||||
|
registerConfigInput := func(kind, path, checksum, source string) {
|
||||||
|
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum, Source: source})
|
||||||
|
}
|
||||||
|
registerSessionConfigInput := func(path, checksum string) {
|
||||||
|
source := env.Config.SessionSource
|
||||||
|
if strings.TrimSpace(source.Source) == "" {
|
||||||
|
source.Source = "session_config"
|
||||||
|
}
|
||||||
|
inputs = append(inputs, manifest.InputRecord{
|
||||||
|
Kind: "session_config",
|
||||||
|
Path: path,
|
||||||
|
Checksum: checksum,
|
||||||
|
Source: source.Source,
|
||||||
|
S3Bucket: source.S3Bucket,
|
||||||
|
S3Key: source.S3Key,
|
||||||
|
S3Size: source.S3Size,
|
||||||
|
S3ETag: source.S3ETag,
|
||||||
|
SpoolPath: source.SpoolPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
campaignDst := filepath.Join(paths.InputsDir, "campaign.yml")
|
||||||
|
campaignChecksum, err := copyFileIfChanged(env.ArtifactStore, campaignSrc, campaignDst)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: materialize campaign.yml: %w", err)
|
||||||
|
}
|
||||||
|
registerConfigInput("campaign_config", campaignDst, campaignChecksum, "campaign_config")
|
||||||
|
|
||||||
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
sessionDst := filepath.Join(paths.InputsDir, "session.yml")
|
||||||
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
sessionChecksum, err := copyFileIfChanged(env.ArtifactStore, sessionSrc, sessionDst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
return nil, fmt.Errorf("prepare: materialize session.yml: %w", err)
|
||||||
}
|
}
|
||||||
registerInput("session_config", sessionDst, sessionChecksum)
|
registerSessionConfigInput(sessionDst, sessionChecksum)
|
||||||
|
|
||||||
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
pipelineResolvedBytes, err := renderResolvedPipeline(env.Config.Pipeline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -123,16 +160,17 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
kind string
|
kind string
|
||||||
src string
|
src string
|
||||||
dst string
|
dst string
|
||||||
|
source string
|
||||||
}{
|
}{
|
||||||
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml")},
|
{kind: "speakers", src: speakersSrc, dst: filepath.Join(paths.InputsDir, "speakers.yml"), source: speakersInput.Source},
|
||||||
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml")},
|
{kind: "autocorrect", src: autocorrectSrc, dst: filepath.Join(paths.InputsDir, "autocorrect.yml"), source: autocorrectInput.Source},
|
||||||
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml")},
|
{kind: "glossary", src: glossarySrc, dst: filepath.Join(paths.InputsDir, "glossary.yml"), source: glossaryInput.Source},
|
||||||
} {
|
} {
|
||||||
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
checksum, err := copyFileIfChanged(env.ArtifactStore, cfgFile.src, cfgFile.dst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
return nil, fmt.Errorf("prepare: materialize %s: %w", cfgFile.kind, err)
|
||||||
}
|
}
|
||||||
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
registerConfigInput(cfgFile.kind, cfgFile.dst, checksum, cfgFile.source)
|
||||||
}
|
}
|
||||||
|
|
||||||
if useS3Audio {
|
if useS3Audio {
|
||||||
@@ -145,6 +183,20 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
previousRequirements := collectPreparePreviousRequirements(env.Config)
|
||||||
|
var previousHydration *previousSessionHydrationResult
|
||||||
|
if len(previousRequirements) > 0 {
|
||||||
|
if err := clearManagedPreviousState(paths); err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: clear previous-session cache: %w", err)
|
||||||
|
}
|
||||||
|
hydration, err := hydratePreviousSessionArtifacts(ctx, env, paths, previousRequirements)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("prepare: hydrate previous-session artifacts: %w", err)
|
||||||
|
}
|
||||||
|
previousHydration = hydration
|
||||||
|
inputs = append(inputs, hydration.Inputs...)
|
||||||
|
}
|
||||||
|
|
||||||
sort.Slice(inputs, func(i, j int) bool {
|
sort.Slice(inputs, func(i, j int) bool {
|
||||||
if inputs[i].Kind != inputs[j].Kind {
|
if inputs[i].Kind != inputs[j].Kind {
|
||||||
return inputs[i].Kind < inputs[j].Kind
|
return inputs[i].Kind < inputs[j].Kind
|
||||||
@@ -153,13 +205,27 @@ func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
})
|
})
|
||||||
m.Inputs = inputs
|
m.Inputs = inputs
|
||||||
|
|
||||||
return &StageResult{
|
metadata := map[string]any{
|
||||||
Metadata: map[string]any{
|
|
||||||
"prepared": true,
|
"prepared": true,
|
||||||
"stage": "prepare",
|
"stage": "prepare",
|
||||||
"inputs_count": len(inputs),
|
"inputs_count": len(inputs),
|
||||||
"audio_files_resolved": countAudioInputs(inputs),
|
"audio_files_resolved": countAudioInputs(inputs),
|
||||||
},
|
}
|
||||||
|
if len(previousRequirements) > 0 {
|
||||||
|
metadata["previous_requirements_count"] = len(previousRequirements)
|
||||||
|
if previousHydration != nil {
|
||||||
|
metadata["previous_artifacts_hydrated"] = append([]string(nil), previousHydration.Hydrated...)
|
||||||
|
metadata["previous_artifacts_hydrated_count"] = len(previousHydration.Hydrated)
|
||||||
|
metadata["previous_artifacts_missing_optional"] = append([]string(nil), previousHydration.SkippedMissing...)
|
||||||
|
metadata["previous_artifacts_missing_optional_count"] = len(previousHydration.SkippedMissing)
|
||||||
|
if strings.TrimSpace(previousHydration.PreviousRunID) != "" {
|
||||||
|
metadata["previous_session_run_id"] = strings.TrimSpace(previousHydration.PreviousRunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StageResult{
|
||||||
|
Metadata: metadata,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,6 +233,31 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
|
|||||||
return yaml.Marshal(cfg)
|
return yaml.Marshal(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stableInputSource(resolved config.ResolvedInputFile, fallbackPath, fallbackConfigPath string) config.ResolvedInputFile {
|
||||||
|
if strings.TrimSpace(resolved.Path) != "" || strings.TrimSpace(resolved.ConfigPath) != "" || strings.TrimSpace(resolved.Source) != "" {
|
||||||
|
if strings.TrimSpace(resolved.ConfigPath) == "" {
|
||||||
|
resolved.ConfigPath = fallbackConfigPath
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(resolved.Source) == "" {
|
||||||
|
resolved.Source = "session_config"
|
||||||
|
}
|
||||||
|
return resolved
|
||||||
|
}
|
||||||
|
return config.ResolvedInputFile{
|
||||||
|
Path: fallbackPath,
|
||||||
|
ConfigPath: fallbackConfigPath,
|
||||||
|
Source: "session_config",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||||
|
basePath := strings.TrimSpace(input.ConfigPath)
|
||||||
|
if basePath == "" {
|
||||||
|
return "", fmt.Errorf("source config path is required")
|
||||||
|
}
|
||||||
|
return resolvePath(filepath.Dir(basePath), input.Path)
|
||||||
|
}
|
||||||
|
|
||||||
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
||||||
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
||||||
if inputs.AudioS3 != nil {
|
if inputs.AudioS3 != nil {
|
||||||
@@ -354,6 +445,35 @@ func countAudioInputs(inputs []manifest.InputRecord) int {
|
|||||||
return count
|
return count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func collectPreparePreviousRequirements(cfg *config.Config) []artifacts.PreviousArtifactRequirement {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return artifacts.CollectPreviousArtifactRequirements(cfg.Pipeline.Scriptorium.Artifacts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func clearManagedPreviousState(paths artifacts.SessionPaths) error {
|
||||||
|
previousDir := filepath.Clean(paths.PreviousDir)
|
||||||
|
sessionRoot := filepath.Clean(paths.Root)
|
||||||
|
if strings.TrimSpace(previousDir) == "" || strings.TrimSpace(sessionRoot) == "" {
|
||||||
|
return fmt.Errorf("previous/session root paths are required")
|
||||||
|
}
|
||||||
|
if previousDir == sessionRoot {
|
||||||
|
return fmt.Errorf("refusing to clear session root as previous cache: %q", previousDir)
|
||||||
|
}
|
||||||
|
prefix := sessionRoot + string(filepath.Separator)
|
||||||
|
if !strings.HasPrefix(previousDir, prefix) {
|
||||||
|
return fmt.Errorf("refusing to clear path outside session root: %q", previousDir)
|
||||||
|
}
|
||||||
|
if filepath.Base(previousDir) != config.PathPreviousDirSegment {
|
||||||
|
return fmt.Errorf("refusing to clear non-previous path %q", previousDir)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(previousDir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.MkdirAll(previousDir, 0o755)
|
||||||
|
}
|
||||||
|
|
||||||
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
|
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
|
||||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
return ""
|
return ""
|
||||||
|
|||||||
475
internal/stage/prepare_previous.go
Normal file
475
internal/stage/prepare_previous.go
Normal file
@@ -0,0 +1,475 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
preparePreviousInputKindManifest = "previous_manifest"
|
||||||
|
preparePreviousInputKindArtifact = "previous_artifact"
|
||||||
|
preparePreviousInputSource = "previous_session_archive.current"
|
||||||
|
)
|
||||||
|
|
||||||
|
// previousSessionHydrationResult captures prepare-time previous-session cache materialization.
|
||||||
|
type previousSessionHydrationResult struct {
|
||||||
|
Inputs []manifest.InputRecord
|
||||||
|
Hydrated []string
|
||||||
|
SkippedMissing []string
|
||||||
|
PreviousRunID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func hydratePreviousSessionArtifacts(
|
||||||
|
ctx context.Context,
|
||||||
|
env *Env,
|
||||||
|
paths artifacts.SessionPaths,
|
||||||
|
requirements []artifacts.PreviousArtifactRequirement,
|
||||||
|
) (*previousSessionHydrationResult, error) {
|
||||||
|
if len(requirements) == 0 {
|
||||||
|
return &previousSessionHydrationResult{}, nil
|
||||||
|
}
|
||||||
|
if env == nil || env.Config == nil || env.Config.Session == nil || env.Config.Pipeline == nil {
|
||||||
|
return nil, fmt.Errorf("resolved config with session/pipeline is required")
|
||||||
|
}
|
||||||
|
if env.ArtifactStore == nil {
|
||||||
|
return nil, fmt.Errorf("artifact store is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
orderedRequirements := append([]artifacts.PreviousArtifactRequirement(nil), requirements...)
|
||||||
|
sort.Slice(orderedRequirements, func(i, j int) bool {
|
||||||
|
return orderedRequirements[i].Name < orderedRequirements[j].Name
|
||||||
|
})
|
||||||
|
|
||||||
|
requiredNames := requiredPreviousArtifactNames(orderedRequirements)
|
||||||
|
optionalNames := optionalPreviousArtifactNames(orderedRequirements)
|
||||||
|
previousSessionID := strings.TrimSpace(env.Config.Session.PreviousSessionID)
|
||||||
|
if previousSessionID == "" {
|
||||||
|
if len(requiredNames) > 0 {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"previous_session_id is required for required previous-session artifacts: %s",
|
||||||
|
strings.Join(requiredNames, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return &previousSessionHydrationResult{SkippedMissing: optionalNames}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if env.ObjectStore == nil {
|
||||||
|
return nil, fmt.Errorf("previous-session artifact hydration requires object store backend")
|
||||||
|
}
|
||||||
|
if env.Config.Pipeline.Storage.S3 == nil {
|
||||||
|
return nil, fmt.Errorf("pipeline.storage.s3 configuration is required for previous-session artifact hydration")
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
return nil, fmt.Errorf("session campaign is required for previous-session artifact hydration")
|
||||||
|
}
|
||||||
|
bucket := strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
|
||||||
|
if bucket == "" {
|
||||||
|
return nil, fmt.Errorf("pipeline.storage.s3.bucket is required for previous-session artifact hydration")
|
||||||
|
}
|
||||||
|
|
||||||
|
previousSessionPrefix := artifacts.S3SessionPrefix(
|
||||||
|
env.Config.Pipeline.Storage.S3.RootPrefix,
|
||||||
|
campaign,
|
||||||
|
previousSessionID,
|
||||||
|
)
|
||||||
|
currentManifestKey, currentRunIDKey := artifacts.ResolveArchiveCurrentStateKeys(previousSessionPrefix)
|
||||||
|
|
||||||
|
result := &previousSessionHydrationResult{}
|
||||||
|
|
||||||
|
runPointerExists, err := env.ObjectStore.Exists(ctx, currentRunIDKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("check previous-session current run pointer %q: %w", currentRunIDKey, err)
|
||||||
|
}
|
||||||
|
if !runPointerExists {
|
||||||
|
if len(requiredNames) > 0 {
|
||||||
|
return nil, fmt.Errorf("required previous-session artifacts unavailable: remote current run pointer missing: %q", currentRunIDKey)
|
||||||
|
}
|
||||||
|
result.SkippedMissing = optionalNames
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
runIDTemp, err := downloadObjectToTempStage(ctx, env.ObjectStore, currentRunIDKey, "narratio-prepare-previous-run-id-*.txt")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("download previous-session current run pointer %q: %w", currentRunIDKey, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(runIDTemp) }()
|
||||||
|
|
||||||
|
runIDBytes, err := os.ReadFile(runIDTemp)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read previous-session current run pointer %q: %w", currentRunIDKey, err)
|
||||||
|
}
|
||||||
|
previousRunID := strings.TrimSpace(string(runIDBytes))
|
||||||
|
if previousRunID == "" {
|
||||||
|
return nil, fmt.Errorf("previous-session current run pointer %q is empty", currentRunIDKey)
|
||||||
|
}
|
||||||
|
result.PreviousRunID = previousRunID
|
||||||
|
|
||||||
|
manifestExists, err := env.ObjectStore.Exists(ctx, currentManifestKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("check previous-session current manifest %q: %w", currentManifestKey, err)
|
||||||
|
}
|
||||||
|
if !manifestExists {
|
||||||
|
if len(requiredNames) > 0 {
|
||||||
|
return nil, fmt.Errorf("required previous-session artifacts unavailable: remote current manifest missing: %q", currentManifestKey)
|
||||||
|
}
|
||||||
|
result.SkippedMissing = optionalNames
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(paths.PreviousManifestPath), 0o755); err != nil {
|
||||||
|
return nil, fmt.Errorf("create previous manifest directory: %w", err)
|
||||||
|
}
|
||||||
|
if err := env.ObjectStore.Download(ctx, currentManifestKey, paths.PreviousManifestPath); err != nil {
|
||||||
|
return nil, fmt.Errorf("download previous-session current manifest %q: %w", currentManifestKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestStore := &manifest.LocalStore{}
|
||||||
|
previousManifest, err := manifestStore.Load(ctx, paths.PreviousManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode downloaded previous-session manifest %q: %w", currentManifestKey, err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(previousManifest.SessionID) != previousSessionID {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"previous-session manifest session_id %q does not match configured previous_session_id %q",
|
||||||
|
strings.TrimSpace(previousManifest.SessionID),
|
||||||
|
previousSessionID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(previousManifest.Campaign) != campaign {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"previous-session manifest campaign %q does not match current campaign %q",
|
||||||
|
strings.TrimSpace(previousManifest.Campaign),
|
||||||
|
campaign,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(previousManifest.RunID) == "" {
|
||||||
|
return nil, fmt.Errorf("previous-session manifest run_id is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(previousManifest.RunID) != previousRunID {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"previous-session current run pointer %q references run %q but current manifest run_id is %q",
|
||||||
|
currentRunIDKey,
|
||||||
|
previousRunID,
|
||||||
|
strings.TrimSpace(previousManifest.RunID),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestChecksum, err := env.ArtifactStore.Checksum(paths.PreviousManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("checksum downloaded previous-session manifest: %w", err)
|
||||||
|
}
|
||||||
|
result.Inputs = append(result.Inputs, manifest.InputRecord{
|
||||||
|
Kind: preparePreviousInputKindManifest,
|
||||||
|
Path: paths.PreviousManifestPath,
|
||||||
|
Checksum: manifestChecksum,
|
||||||
|
Source: preparePreviousInputSource,
|
||||||
|
S3Bucket: bucket,
|
||||||
|
S3Key: currentManifestKey,
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, requirement := range orderedRequirements {
|
||||||
|
candidates := previousArtifactRelativePathCandidates(requirement.Name, previousManifest, env.Config)
|
||||||
|
if len(candidates) == 0 {
|
||||||
|
if requirement.Required {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"required previous-session artifact %q is unavailable in previous-session manifest/archive",
|
||||||
|
requirement.Name,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
result.SkippedMissing = append(result.SkippedMissing, requirement.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
selectedRel := ""
|
||||||
|
selectedKey := ""
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
remoteKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, candidate)
|
||||||
|
exists, err := env.ObjectStore.Exists(ctx, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("check previous-session artifact object %q: %w", remoteKey, err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
selectedRel = candidate
|
||||||
|
selectedKey = remoteKey
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if selectedRel == "" {
|
||||||
|
if requirement.Required {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"required previous-session artifact %q object missing from archive candidate keys",
|
||||||
|
requirement.Name,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
result.SkippedMissing = append(result.SkippedMissing, requirement.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
localPath := artifacts.SessionPreviousArtifactPath(paths, selectedRel)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return nil, fmt.Errorf("create previous-session artifact directory for %q: %w", localPath, err)
|
||||||
|
}
|
||||||
|
if err := env.ObjectStore.Download(ctx, selectedKey, localPath); err != nil {
|
||||||
|
return nil, fmt.Errorf("download previous-session artifact %q from %q: %w", requirement.Name, selectedKey, err)
|
||||||
|
}
|
||||||
|
if err := requireNonEmptyFile(localPath, "previous-session artifact "+requirement.Name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
checksum, err := env.ArtifactStore.Checksum(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("checksum previous-session artifact %q: %w", requirement.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Inputs = append(result.Inputs, manifest.InputRecord{
|
||||||
|
Kind: preparePreviousInputKindArtifact,
|
||||||
|
Path: localPath,
|
||||||
|
Checksum: checksum,
|
||||||
|
Source: preparePreviousInputSource,
|
||||||
|
S3Bucket: bucket,
|
||||||
|
S3Key: selectedKey,
|
||||||
|
})
|
||||||
|
result.Hydrated = append(result.Hydrated, requirement.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(result.Hydrated)
|
||||||
|
sort.Strings(result.SkippedMissing)
|
||||||
|
sort.Slice(result.Inputs, func(i, j int) bool {
|
||||||
|
if result.Inputs[i].Kind != result.Inputs[j].Kind {
|
||||||
|
return result.Inputs[i].Kind < result.Inputs[j].Kind
|
||||||
|
}
|
||||||
|
return result.Inputs[i].Path < result.Inputs[j].Path
|
||||||
|
})
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requiredPreviousArtifactNames(requirements []artifacts.PreviousArtifactRequirement) []string {
|
||||||
|
names := make([]string, 0, len(requirements))
|
||||||
|
for _, requirement := range requirements {
|
||||||
|
if requirement.Required {
|
||||||
|
names = append(names, strings.TrimSpace(requirement.Name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func optionalPreviousArtifactNames(requirements []artifacts.PreviousArtifactRequirement) []string {
|
||||||
|
names := make([]string, 0, len(requirements))
|
||||||
|
for _, requirement := range requirements {
|
||||||
|
if requirement.Required {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names = append(names, strings.TrimSpace(requirement.Name))
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousArtifactRelativePathCandidates(
|
||||||
|
artifactName string,
|
||||||
|
previousManifest *manifest.Manifest,
|
||||||
|
cfg *config.Config,
|
||||||
|
) []string {
|
||||||
|
candidates := []string{}
|
||||||
|
appendCandidate := func(v string) {
|
||||||
|
normalized, err := normalizeArchiveRelativePath(v)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
candidates = append(candidates, normalized)
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceID := artifacts.ConfiguredArtifactSourceID(artifactName)
|
||||||
|
if rel, ok := previousManifestArtifactRelativePathBySourceID(previousManifest, sourceID); ok {
|
||||||
|
appendCandidate(rel)
|
||||||
|
base := path.Base(rel)
|
||||||
|
for _, promoted := range previousManifestPromotedPaths(previousManifest) {
|
||||||
|
if path.Base(promoted) == base {
|
||||||
|
appendCandidate(promoted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Scriptorium != nil {
|
||||||
|
if artifactCfg, ok := cfg.Pipeline.Scriptorium.Artifacts[artifactName]; ok {
|
||||||
|
appendCandidate(artifactCfg.OutputPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dedupeOrderedStrings(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousManifestArtifactRelativePathBySourceID(previousManifest *manifest.Manifest, sourceID string) (string, bool) {
|
||||||
|
if previousManifest == nil || len(previousManifest.Stages) == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
sourceID = strings.TrimSpace(sourceID)
|
||||||
|
if sourceID == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
stageNames := make([]string, 0, len(previousManifest.Stages))
|
||||||
|
if _, ok := previousManifest.Stages["analyze"]; ok {
|
||||||
|
stageNames = append(stageNames, "analyze")
|
||||||
|
}
|
||||||
|
for stageName := range previousManifest.Stages {
|
||||||
|
if stageName == "analyze" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stageNames = append(stageNames, stageName)
|
||||||
|
}
|
||||||
|
start := 0
|
||||||
|
if len(stageNames) > 0 && stageNames[0] == "analyze" {
|
||||||
|
start = 1
|
||||||
|
}
|
||||||
|
sort.Strings(stageNames[start:])
|
||||||
|
|
||||||
|
for _, stageName := range stageNames {
|
||||||
|
sr := previousManifest.Stages[stageName]
|
||||||
|
if sr == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, out := range sr.Outputs {
|
||||||
|
if strings.TrimSpace(out.SourceID) != sourceID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rel, ok := derivePreviousManifestRelativePath(previousManifest, out.LocalPath)
|
||||||
|
if ok {
|
||||||
|
return rel, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func derivePreviousManifestRelativePath(previousManifest *manifest.Manifest, localPath string) (string, bool) {
|
||||||
|
trimmed := strings.TrimSpace(localPath)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(trimmed) {
|
||||||
|
normalized, err := normalizeArchiveRelativePath(filepath.ToSlash(trimmed))
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return normalized, true
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionRoot, ok := previousManifestSessionRoot(previousManifest)
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(sessionRoot, trimmed)
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
normalized, err := normalizeArchiveRelativePath(filepath.ToSlash(rel))
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return normalized, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousManifestSessionRoot(previousManifest *manifest.Manifest) (string, bool) {
|
||||||
|
if previousManifest == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
runRoot := filepath.Clean(strings.TrimSpace(previousManifest.LocalWorkDir))
|
||||||
|
runID := strings.TrimSpace(previousManifest.RunID)
|
||||||
|
if runRoot == "" || runID == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if filepath.Base(runRoot) != runID {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
runsDir := filepath.Dir(runRoot)
|
||||||
|
if filepath.Base(runsDir) != config.PathRunsDirSegment {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return filepath.Dir(runsDir), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousManifestPromotedPaths(previousManifest *manifest.Manifest) []string {
|
||||||
|
if previousManifest == nil || len(previousManifest.Stages) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sr := previousManifest.Stages["archive"]
|
||||||
|
if sr == nil || sr.Metadata == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, ok := sr.Metadata["promoted_paths"]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
values, ok := raw.([]any)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
asString, ok := value.(string)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
normalized, err := normalizeArchiveRelativePath(asString)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, normalized)
|
||||||
|
}
|
||||||
|
return dedupeOrderedStrings(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func dedupeOrderedStrings(values []string) []string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := make([]string, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[trimmed]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[trimmed] = struct{}{}
|
||||||
|
out = append(out, trimmed)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadObjectToTempStage(
|
||||||
|
ctx context.Context,
|
||||||
|
store interface {
|
||||||
|
Download(context.Context, string, string) error
|
||||||
|
},
|
||||||
|
key, pattern string,
|
||||||
|
) (string, error) {
|
||||||
|
tmp, err := os.CreateTemp("", pattern)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := tmp.Name()
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
return "", fmt.Errorf("close temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := store.Download(ctx, key, path); err != nil {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
415
internal/stage/prepare_previous_test.go
Normal file
415
internal/stage/prepare_previous_test.go
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsDownloadsManifestAndRequiredArtifact(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeArtifactObject: true,
|
||||||
|
artifactBody: "# previous recap\n",
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("result is nil")
|
||||||
|
}
|
||||||
|
if len(result.Inputs) != 2 {
|
||||||
|
t.Fatalf("inputs len = %d, want 2", len(result.Inputs))
|
||||||
|
}
|
||||||
|
if !containsString(result.Hydrated, "session_recap") {
|
||||||
|
t.Fatalf("hydrated = %#v, want session_recap", result.Hydrated)
|
||||||
|
}
|
||||||
|
if len(result.SkippedMissing) != 0 {
|
||||||
|
t.Fatalf("skipped missing = %#v, want none", result.SkippedMissing)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(sessionPaths.PreviousManifestPath); err != nil {
|
||||||
|
t.Fatalf("previous manifest missing: %v", err)
|
||||||
|
}
|
||||||
|
recapPath := artifacts.SessionPreviousArtifactPath(sessionPaths, "artifacts/session_recap.md")
|
||||||
|
if _, err := os.Stat(recapPath); err != nil {
|
||||||
|
t.Fatalf("previous artifact missing: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
|
||||||
|
if manifestInput == nil {
|
||||||
|
t.Fatalf("missing input kind %q", preparePreviousInputKindManifest)
|
||||||
|
}
|
||||||
|
if manifestInput.Source != preparePreviousInputSource {
|
||||||
|
t.Fatalf("manifest input source = %q, want %q", manifestInput.Source, preparePreviousInputSource)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactInput := findInputByKind(result.Inputs, preparePreviousInputKindArtifact)
|
||||||
|
if artifactInput == nil {
|
||||||
|
t.Fatalf("missing input kind %q", preparePreviousInputKindArtifact)
|
||||||
|
}
|
||||||
|
if artifactInput.Source != preparePreviousInputSource {
|
||||||
|
t.Fatalf("artifact input source = %q, want %q", artifactInput.Source, preparePreviousInputSource)
|
||||||
|
}
|
||||||
|
if artifactInput.Path != recapPath {
|
||||||
|
t.Fatalf("artifact input path = %q, want %q", artifactInput.Path, recapPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsSkipsMissingOptionalArtifact(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: false},
|
||||||
|
}
|
||||||
|
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeManifestObject: true,
|
||||||
|
includeArtifactObject: false,
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("result is nil")
|
||||||
|
}
|
||||||
|
if !containsString(result.SkippedMissing, "session_recap") {
|
||||||
|
t.Fatalf("skipped missing = %#v, want session_recap", result.SkippedMissing)
|
||||||
|
}
|
||||||
|
if len(result.Hydrated) != 0 {
|
||||||
|
t.Fatalf("hydrated = %#v, want none", result.Hydrated)
|
||||||
|
}
|
||||||
|
manifestInput := findInputByKind(result.Inputs, preparePreviousInputKindManifest)
|
||||||
|
if manifestInput == nil {
|
||||||
|
t.Fatalf("missing input kind %q", preparePreviousInputKindManifest)
|
||||||
|
}
|
||||||
|
if findInputByKind(result.Inputs, preparePreviousInputKindArtifact) != nil {
|
||||||
|
t.Fatalf("unexpected %q input for missing optional artifact", preparePreviousInputKindArtifact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsFailsMissingRequiredArtifact(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeArtifactObject: false,
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "required previous-session artifact") {
|
||||||
|
t.Fatalf("error = %v, want required artifact failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsFailsRequiredWhenPreviousSessionIDUnset(t *testing.T) {
|
||||||
|
env, sessionPaths, _ := previousHydrationFixture(t)
|
||||||
|
env.Config.Session.PreviousSessionID = ""
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "previous_session_id is required") {
|
||||||
|
t.Fatalf("error = %v, want previous_session_id required failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsOptionalWithNoPreviousSessionID(t *testing.T) {
|
||||||
|
env, sessionPaths, _ := previousHydrationFixture(t)
|
||||||
|
env.Config.Session.PreviousSessionID = ""
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
t.Fatal("result is nil")
|
||||||
|
}
|
||||||
|
if len(result.Inputs) != 0 {
|
||||||
|
t.Fatalf("inputs len = %d, want 0", len(result.Inputs))
|
||||||
|
}
|
||||||
|
if !containsString(result.SkippedMissing, "session_recap") {
|
||||||
|
t.Fatalf("skipped missing = %#v, want session_recap", result.SkippedMissing)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sessionPaths.PreviousManifestPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("previous manifest should not be created, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsRespectsCurrentCommitMarker(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: false,
|
||||||
|
includeArtifactObject: true,
|
||||||
|
artifactBody: "# previous recap\n",
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "remote current run pointer missing") {
|
||||||
|
t.Fatalf("error = %v, want current run pointer missing failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsDoesNotUseLocalPreviousWorkspaceState(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeManifestObject: true,
|
||||||
|
includeArtifactObject: false,
|
||||||
|
})
|
||||||
|
// Write a local previous-session workspace file that should be ignored.
|
||||||
|
writeFile(t, filepath.Join(seed.PreviousSessionRoot, "artifacts", "session_recap.md"), "# local stale recap\n")
|
||||||
|
|
||||||
|
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "object missing from archive") {
|
||||||
|
t.Fatalf("error = %v, want remote-object-missing failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydratePreviousSessionArtifactsUsesExplicitStorageKeys(t *testing.T) {
|
||||||
|
env, sessionPaths, fake := previousHydrationFixture(t)
|
||||||
|
requirements := []artifacts.PreviousArtifactRequirement{
|
||||||
|
{Name: "session_recap", Required: true},
|
||||||
|
}
|
||||||
|
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeArtifactObject: true,
|
||||||
|
artifactBody: "# previous recap\n",
|
||||||
|
})
|
||||||
|
|
||||||
|
capture := &preparePreviousCaptureStore{delegate: fake}
|
||||||
|
env.ObjectStore = capture
|
||||||
|
|
||||||
|
_, err := hydratePreviousSessionArtifacts(context.Background(), env, sessionPaths, requirements)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("hydratePreviousSessionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !containsString(capture.existsKeys, seed.RunPointerKey) {
|
||||||
|
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.RunPointerKey)
|
||||||
|
}
|
||||||
|
if !containsString(capture.existsKeys, seed.ManifestKey) {
|
||||||
|
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.ManifestKey)
|
||||||
|
}
|
||||||
|
if !containsString(capture.existsKeys, seed.ArtifactKey) {
|
||||||
|
t.Fatalf("exists keys = %#v, want %q", capture.existsKeys, seed.ArtifactKey)
|
||||||
|
}
|
||||||
|
if !containsString(capture.downloadKeys, seed.RunPointerKey) {
|
||||||
|
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.RunPointerKey)
|
||||||
|
}
|
||||||
|
if !containsString(capture.downloadKeys, seed.ManifestKey) {
|
||||||
|
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.ManifestKey)
|
||||||
|
}
|
||||||
|
if !containsString(capture.downloadKeys, seed.ArtifactKey) {
|
||||||
|
t.Fatalf("download keys = %#v, want %q", capture.downloadKeys, seed.ArtifactKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type previousStateSeedResult struct {
|
||||||
|
PreviousSessionPrefix string
|
||||||
|
PreviousSessionRoot string
|
||||||
|
ManifestKey string
|
||||||
|
RunPointerKey string
|
||||||
|
ArtifactKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
type previousStateSeedOptions struct {
|
||||||
|
includeRunPointerObject bool
|
||||||
|
includeManifestObject bool
|
||||||
|
includeArtifactObject bool
|
||||||
|
artifactBody string
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedPreviousCurrentState(
|
||||||
|
t *testing.T,
|
||||||
|
env *Env,
|
||||||
|
fake *storage.FakeBackend,
|
||||||
|
options previousStateSeedOptions,
|
||||||
|
) previousStateSeedResult {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if !options.includeRunPointerObject && !options.includeManifestObject && !options.includeArtifactObject {
|
||||||
|
// Keep default behavior deterministic when caller omits explicit flags.
|
||||||
|
options.includeRunPointerObject = true
|
||||||
|
options.includeManifestObject = true
|
||||||
|
}
|
||||||
|
if options.includeManifestObject == false && options.includeArtifactObject {
|
||||||
|
options.includeManifestObject = true
|
||||||
|
}
|
||||||
|
|
||||||
|
previousSessionID := strings.TrimSpace(env.Config.Session.PreviousSessionID)
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
rootPrefix := strings.TrimSpace(env.Config.Pipeline.Storage.S3.RootPrefix)
|
||||||
|
previousSessionPrefix := artifacts.S3SessionPrefix(rootPrefix, campaign, previousSessionID)
|
||||||
|
manifestKey, runPointerKey := artifacts.ResolveArchiveCurrentStateKeys(previousSessionPrefix)
|
||||||
|
|
||||||
|
previousRunID := "20260510T010203Z-a1b2c3d4"
|
||||||
|
previousSessionRoot := filepath.Join(t.TempDir(), "work", campaign, previousSessionID)
|
||||||
|
artifactLocalPath := filepath.Join(previousSessionRoot, "artifacts", "session_recap.md")
|
||||||
|
previousManifest := buildPreviousManifestForSeed(
|
||||||
|
t,
|
||||||
|
previousSessionID,
|
||||||
|
campaign,
|
||||||
|
previousRunID,
|
||||||
|
filepath.Join(previousSessionRoot, "runs", previousRunID),
|
||||||
|
artifactLocalPath,
|
||||||
|
)
|
||||||
|
|
||||||
|
if options.includeRunPointerObject || (!options.includeRunPointerObject && !options.includeManifestObject && !options.includeArtifactObject) {
|
||||||
|
fake.SeedObject(storage.FakeObject{
|
||||||
|
Key: runPointerKey,
|
||||||
|
Data: []byte(previousRunID + "\n"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if options.includeManifestObject || options.includeArtifactObject {
|
||||||
|
fake.SeedObject(storage.FakeObject{
|
||||||
|
Key: manifestKey,
|
||||||
|
Data: previousManifest,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, "artifacts/session_recap.md")
|
||||||
|
if options.includeArtifactObject {
|
||||||
|
body := options.artifactBody
|
||||||
|
if body == "" {
|
||||||
|
body = "# previous recap\n"
|
||||||
|
}
|
||||||
|
fake.SeedObject(storage.FakeObject{
|
||||||
|
Key: artifactKey,
|
||||||
|
Data: []byte(body),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return previousStateSeedResult{
|
||||||
|
PreviousSessionPrefix: previousSessionPrefix,
|
||||||
|
PreviousSessionRoot: previousSessionRoot,
|
||||||
|
ManifestKey: manifestKey,
|
||||||
|
RunPointerKey: runPointerKey,
|
||||||
|
ArtifactKey: artifactKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func previousHydrationFixture(t *testing.T) (*Env, artifacts.SessionPaths, *storage.FakeBackend) {
|
||||||
|
t.Helper()
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.PreviousSessionID = "2026-05-10"
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
RootPrefix: "dnd",
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {
|
||||||
|
Enabled: true,
|
||||||
|
OutputPath: "artifacts/session_recap.md",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
env.ObjectStore = fake
|
||||||
|
|
||||||
|
paths, err := ensureLayoutForEnv(env, m.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ensureLayoutForEnv() error = %v", err)
|
||||||
|
}
|
||||||
|
return env, paths, fake
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildPreviousManifestForSeed(
|
||||||
|
t *testing.T,
|
||||||
|
sessionID, campaign, runID, runRoot, artifactPath string,
|
||||||
|
) []byte {
|
||||||
|
t.Helper()
|
||||||
|
now := time.Date(2026, 5, 19, 22, 0, 0, 0, time.UTC)
|
||||||
|
m := manifest.New(sessionID, now)
|
||||||
|
m.Campaign = campaign
|
||||||
|
m.RunID = runID
|
||||||
|
m.LocalWorkDir = runRoot
|
||||||
|
m.MarkStageSucceeded("analyze", now, []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "scriptorium_artifact",
|
||||||
|
SourceID: artifacts.ConfiguredArtifactSourceID("session_recap"),
|
||||||
|
LocalPath: artifactPath,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
m.MarkStageSucceeded("archive", now, nil)
|
||||||
|
m.Stages["archive"].Metadata = map[string]any{
|
||||||
|
"promoted_paths": []string{"artifacts/session_recap.md"},
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(m, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal manifest: %v", err)
|
||||||
|
}
|
||||||
|
return append(data, '\n')
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparePreviousCaptureStore struct {
|
||||||
|
delegate storage.ObjectStore
|
||||||
|
existsKeys []string
|
||||||
|
downloadKeys []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *preparePreviousCaptureStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||||
|
return s.delegate.List(ctx, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *preparePreviousCaptureStore) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
s.downloadKeys = append(s.downloadKeys, key)
|
||||||
|
return s.delegate.Download(ctx, key, localPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *preparePreviousCaptureStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
|
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *preparePreviousCaptureStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
s.existsKeys = append(s.existsKeys, key)
|
||||||
|
return s.delegate.Exists(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findInputByKind(inputs []manifest.InputRecord, kind string) *manifest.InputRecord {
|
||||||
|
for i := range inputs {
|
||||||
|
if inputs[i].Kind == kind {
|
||||||
|
return &inputs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsString(values []string, target string) bool {
|
||||||
|
for _, value := range values {
|
||||||
|
if value == target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -35,6 +35,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
|||||||
|
|
||||||
paths := sessionPathsForEnv(env, m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
for _, p := range []string{
|
for _, p := range []string{
|
||||||
|
filepath.Join(paths.InputsDir, "campaign.yml"),
|
||||||
filepath.Join(paths.InputsDir, "session.yml"),
|
filepath.Join(paths.InputsDir, "session.yml"),
|
||||||
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
||||||
filepath.Join(paths.InputsDir, "speakers.yml"),
|
filepath.Join(paths.InputsDir, "speakers.yml"),
|
||||||
@@ -48,8 +49,8 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(m.Inputs) != 7 {
|
if len(m.Inputs) != 8 {
|
||||||
t.Fatalf("manifest inputs len = %d, want 7", len(m.Inputs))
|
t.Fatalf("manifest inputs len = %d, want 8", len(m.Inputs))
|
||||||
}
|
}
|
||||||
for _, in := range m.Inputs {
|
for _, in := range m.Inputs {
|
||||||
if in.Checksum == "" {
|
if in.Checksum == "" {
|
||||||
@@ -151,6 +152,64 @@ func TestPrepareStageIdempotent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRecordsLocalSessionProvenance(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||||
|
if sessionInput.Source != "session_config" {
|
||||||
|
t.Fatalf("session source = %q, want session_config", sessionInput.Source)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Bucket != "" || sessionInput.S3Key != "" || sessionInput.SpoolPath != "" {
|
||||||
|
t.Fatalf("local session input has unexpected remote provenance: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRecordsRemoteSessionProvenance(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
remoteSessionPath := filepath.Join(t.TempDir(), "downloaded-session.yml")
|
||||||
|
writeFile(t, remoteSessionPath, "session_id: 2026-05-03\ninputs:\n audio_s3:\n prefix: audio/\n")
|
||||||
|
env.Config.SessionPath = remoteSessionPath
|
||||||
|
env.Config.Session.Inputs.AudioDir = ""
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
env.Config.SessionSource = config.SessionSource{
|
||||||
|
Source: "session_config.s3",
|
||||||
|
LocalPath: remoteSessionPath,
|
||||||
|
S3Bucket: "my-dnd-archive",
|
||||||
|
S3Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml",
|
||||||
|
S3Size: 58,
|
||||||
|
S3ETag: "session-etag",
|
||||||
|
SpoolPath: remoteSessionPath,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "sample-campaign", m.SessionID, m.RunID)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||||
|
env.ObjectStore = fake
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
sessionInput := findManifestInput(t, m.Inputs, "session_config")
|
||||||
|
if sessionInput.Source != "session_config.s3" {
|
||||||
|
t.Fatalf("session source = %q, want session_config.s3", sessionInput.Source)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Bucket != "my-dnd-archive" || sessionInput.S3Key != "dnd/campaigns/sample-campaign/sessions/2026-05-03/session.yml" {
|
||||||
|
t.Fatalf("remote session input missing bucket/key: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
if sessionInput.S3Size != 58 || sessionInput.S3ETag != "session-etag" || sessionInput.SpoolPath != remoteSessionPath {
|
||||||
|
t.Fatalf("remote session input missing metadata: %#v", sessionInput)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||||
env, m := setupPrepareEnv(t)
|
env, m := setupPrepareEnv(t)
|
||||||
env.Config.Session.Campaign = "forsaken"
|
env.Config.Session.Campaign = "forsaken"
|
||||||
@@ -284,6 +343,188 @@ func TestPrepareStageAudioSourceConflictFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageWithoutPreviousRequirementsDoesNotTouchPreviousState(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
stalePath := filepath.Join(paths.PreviousDir, "stale.txt")
|
||||||
|
writeFile(t, stalePath, "stale")
|
||||||
|
|
||||||
|
_, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(stalePath); err != nil {
|
||||||
|
t.Fatalf("expected previous stale file to remain untouched: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageOptionalPreviousArtifactWithoutPreviousSessionIDSucceeds(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
configurePreparePreviousArtifactSource(env, false)
|
||||||
|
env.Config.Session.PreviousSessionID = ""
|
||||||
|
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeFile(t, filepath.Join(paths.PreviousArtifactsDir, "stale.txt"), "stale")
|
||||||
|
|
||||||
|
result, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_requirements_count"] != 1 {
|
||||||
|
t.Fatalf("metadata previous_requirements_count = %#v, want 1", result.Metadata["previous_requirements_count"])
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_artifacts_hydrated_count"] != 0 {
|
||||||
|
t.Fatalf("metadata previous_artifacts_hydrated_count = %#v, want 0", result.Metadata["previous_artifacts_hydrated_count"])
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_artifacts_missing_optional_count"] != 1 {
|
||||||
|
t.Fatalf("metadata previous_artifacts_missing_optional_count = %#v, want 1", result.Metadata["previous_artifacts_missing_optional_count"])
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(paths.PreviousArtifactsDir, "stale.txt")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected stale previous state to be cleared, stat err = %v", err)
|
||||||
|
}
|
||||||
|
for _, in := range m.Inputs {
|
||||||
|
if in.Kind == preparePreviousInputKindManifest || in.Kind == preparePreviousInputKindArtifact {
|
||||||
|
t.Fatalf("unexpected previous input record: %#v", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRequiredPreviousArtifactWithoutPreviousSessionIDFails(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
configurePreparePreviousArtifactSource(env, true)
|
||||||
|
env.Config.Session.PreviousSessionID = ""
|
||||||
|
|
||||||
|
_, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "previous_session_id is required") {
|
||||||
|
t.Fatalf("error = %v, want previous_session_id required failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageHydratesRequiredPreviousArtifactAndRecordsInputs(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.PreviousSessionID = "2026-05-10"
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
configurePreparePreviousArtifactSource(env, true)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
env.ObjectStore = fake
|
||||||
|
seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeArtifactObject: true,
|
||||||
|
artifactBody: "# prior recap\n",
|
||||||
|
})
|
||||||
|
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
result, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(paths.PreviousManifestPath); err != nil {
|
||||||
|
t.Fatalf("expected previous manifest: %v", err)
|
||||||
|
}
|
||||||
|
recapPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
if _, err := os.Stat(recapPath); err != nil {
|
||||||
|
t.Fatalf("expected previous artifact: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var hasPreviousManifest, hasPreviousArtifact bool
|
||||||
|
for _, in := range m.Inputs {
|
||||||
|
if in.Kind == preparePreviousInputKindManifest {
|
||||||
|
hasPreviousManifest = true
|
||||||
|
}
|
||||||
|
if in.Kind == preparePreviousInputKindArtifact {
|
||||||
|
hasPreviousArtifact = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !hasPreviousManifest || !hasPreviousArtifact {
|
||||||
|
t.Fatalf("manifest inputs missing previous provenance: %#v", m.Inputs)
|
||||||
|
}
|
||||||
|
if result.Metadata["previous_artifacts_hydrated_count"] != 1 {
|
||||||
|
t.Fatalf("metadata previous_artifacts_hydrated_count = %#v, want 1", result.Metadata["previous_artifacts_hydrated_count"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageRerunOverwritesPreviousCache(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioFiles = []string{"./audio/a.flac"}
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.PreviousSessionID = "2026-05-10"
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
configurePreparePreviousArtifactSource(env, true)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
env.ObjectStore = fake
|
||||||
|
seed := seedPreviousCurrentState(t, env, fake, previousStateSeedOptions{
|
||||||
|
includeRunPointerObject: true,
|
||||||
|
includeArtifactObject: true,
|
||||||
|
artifactBody: "# old recap\n",
|
||||||
|
})
|
||||||
|
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("first prepare run error = %v", err)
|
||||||
|
}
|
||||||
|
recapPath := artifacts.SessionPreviousArtifactPath(paths, "artifacts/session_recap.md")
|
||||||
|
firstBytes, err := os.ReadFile(recapPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read first hydrated artifact: %v", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(firstBytes)) != "# old recap" {
|
||||||
|
t.Fatalf("first hydrated content = %q, want %q", strings.TrimSpace(string(firstBytes)), "# old recap")
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFile(t, filepath.Join(paths.PreviousArtifactsDir, "stale.txt"), "stale")
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: seed.ArtifactKey, Data: []byte("# new recap\n")})
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("second prepare run error = %v", err)
|
||||||
|
}
|
||||||
|
secondBytes, err := os.ReadFile(recapPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read second hydrated artifact: %v", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(secondBytes)) != "# new recap" {
|
||||||
|
t.Fatalf("second hydrated content = %q, want %q", strings.TrimSpace(string(secondBytes)), "# new recap")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(paths.PreviousArtifactsDir, "stale.txt")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected stale previous cache file to be removed, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func configurePreparePreviousArtifactSource(env *Env, required bool) {
|
||||||
|
env.Config.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {
|
||||||
|
Enabled: true,
|
||||||
|
OutputPath: "artifacts/session_recap.md",
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"previous_recap": {
|
||||||
|
Source: "narratio.previous_session.artifact.session_recap",
|
||||||
|
Required: required,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
@@ -291,8 +532,15 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
|
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
|
|
||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
|
writeFile(t, campaignPath, `campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`)
|
||||||
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
@@ -300,16 +548,32 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
|
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}},
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
Campaign: "sample-campaign",
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
},
|
||||||
AutocorrectFile: "./autocorrect.yml",
|
},
|
||||||
GlossaryFile: "./glossary.yml",
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{
|
||||||
|
Path: "./speakers.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{
|
||||||
|
Path: "./autocorrect.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
|
},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{
|
||||||
|
Path: "./glossary.yml",
|
||||||
|
ConfigPath: campaignPath,
|
||||||
|
Source: "campaign_config",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -337,3 +601,14 @@ func snapshotInputs(inputs []manifest.InputRecord) map[string]string {
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func findManifestInput(t *testing.T, inputs []manifest.InputRecord, kind string) manifest.InputRecord {
|
||||||
|
t.Helper()
|
||||||
|
for _, input := range inputs {
|
||||||
|
if input.Kind == kind {
|
||||||
|
return input
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("manifest input kind %q not found in %#v", kind, inputs)
|
||||||
|
return manifest.InputRecord{}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -99,6 +100,9 @@ func runLocalPathForCanonical(layout runStageLayout, sessionPaths artifacts.Sess
|
|||||||
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
return "", fmt.Errorf("canonical path %q is outside session root %q", cleanCanonical, sessionPaths.Root)
|
return "", fmt.Errorf("canonical path %q is outside session root %q", cleanCanonical, sessionPaths.Root)
|
||||||
}
|
}
|
||||||
|
if rel == config.PathPreviousDirSegment || strings.HasPrefix(rel, config.PathPreviousDirSegment+string(filepath.Separator)) {
|
||||||
|
return cleanCanonical, nil
|
||||||
|
}
|
||||||
localPath := filepath.Join(layout.OutputsDir, rel)
|
localPath := filepath.Join(layout.OutputsDir, rel)
|
||||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
return "", fmt.Errorf("create run-local output parent for %q: %w", localPath, err)
|
return "", fmt.Errorf("create run-local output parent for %q: %w", localPath, err)
|
||||||
|
|||||||
@@ -33,3 +33,24 @@ func TestRunLocalPathForCanonicalCreatesParentDirectories(t *testing.T) {
|
|||||||
t.Fatalf("expected run-local parent directory to exist: %v", err)
|
t.Fatalf("expected run-local parent directory to exist: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunLocalPathForCanonicalKeepsPreviousStateSessionDurable(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sessionRoot := filepath.Join(root, "work", "dilfs", "2026-05-17")
|
||||||
|
layout := runStageLayout{
|
||||||
|
Enabled: true,
|
||||||
|
OutputsDir: filepath.Join(sessionRoot, "runs", "run-1", "prepare", "outputs"),
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(layout.OutputsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir outputs dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canonical := filepath.Join(sessionRoot, "previous", "artifacts", "session_recap.md")
|
||||||
|
got, err := runLocalPathForCanonical(layout, artifacts.SessionPaths{Root: sessionRoot}, canonical)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runLocalPathForCanonical() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != canonical {
|
||||||
|
t.Fatalf("runLocalPathForCanonical() = %q, want canonical %q", got, canonical)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -233,8 +233,10 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
|
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
campaignPath := filepath.Join(cfgDir, "campaign.yml")
|
||||||
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
|
writeFile(t, campaignPath, "campaign: sample-campaign\ninputs:\n speakers_file: ./speakers.yml\n autocorrect_file: ./autocorrect.yml\n glossary_file: ./glossary.yml\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
@@ -243,7 +245,9 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
concurrency := 2
|
concurrency := 2
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
|
Campaign: &config.CampaignConfig{Campaign: "sample-campaign"},
|
||||||
Pipeline: &config.PipelineConfig{
|
Pipeline: &config.PipelineConfig{
|
||||||
Workspace: config.WorkspaceConfig{Root: workspace},
|
Workspace: config.WorkspaceConfig{Root: workspace},
|
||||||
WhisperX: config.WhisperXConfig{
|
WhisperX: config.WhisperXConfig{
|
||||||
@@ -265,6 +269,11 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
GlossaryFile: "./glossary.yml",
|
GlossaryFile: "./glossary.yml",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
StableInputs: config.ResolvedStableInputs{
|
||||||
|
SpeakersFile: config.ResolvedInputFile{Path: "./speakers.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
AutocorrectFile: config.ResolvedInputFile{Path: "./autocorrect.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
GlossaryFile: config.ResolvedInputFile{Path: "./glossary.yml", ConfigPath: campaignPath, Source: "campaign_config"},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
env := &Env{
|
env := &Env{
|
||||||
|
|||||||
Reference in New Issue
Block a user