Compare commits
3 Commits
f050b9dd54
...
71395bb076
| Author | SHA1 | Date | |
|---|---|---|---|
| 71395bb076 | |||
| 79737edf79 | |||
| df2c765b7f |
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
Narratio is a Go orchestration application that turns D&D session audio into polished transcripts and generated session artifacts.
|
Narratio is a Go orchestration application that turns D&D session audio into polished transcripts and generated session artifacts.
|
||||||
|
|
||||||
It coordinates transcription, merge/polish/normalize/trim processing, artifact generation, archive publishing, and resumable run state in one operator workflow.
|
It coordinates transcription, merge/polish/normalize/trim processing, artifact generation, publish-stage uploads, and resumable run state in one operator workflow.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run 2026-04-04
|
narratio run 2026-04-04
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ It coordinates specialized downstream systems rather than reimplementing their d
|
|||||||
- Audita handles transcript correction and polishing.
|
- Audita handles transcript correction and polishing.
|
||||||
- Scriptorium handles prompt execution and generated artifacts.
|
- Scriptorium handles prompt execution and generated artifacts.
|
||||||
|
|
||||||
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and archive semantics.
|
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics.
|
||||||
|
|
||||||
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
|
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ It should record:
|
|||||||
- input and output refs;
|
- input and output refs;
|
||||||
- logs and generated config refs;
|
- logs and generated config refs;
|
||||||
- checksums or provenance where useful;
|
- checksums or provenance where useful;
|
||||||
- non-secret adapter and archive metadata.
|
- non-secret adapter and publish metadata.
|
||||||
|
|
||||||
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
|
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
|
||||||
|
|
||||||
@@ -117,19 +117,19 @@ Narratio should not become a secondary configuration system for downstream tools
|
|||||||
|
|
||||||
Local and remote paths are part of Narratio’s application contract.
|
Local and remote paths are part of Narratio’s application contract.
|
||||||
|
|
||||||
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and archive paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
|
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
|
||||||
|
|
||||||
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
|
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
|
||||||
|
|
||||||
## Archive Invariants
|
## Publish Invariants
|
||||||
|
|
||||||
Archive behavior must preserve a clear commit boundary.
|
Publish behavior must preserve a clear commit boundary.
|
||||||
|
|
||||||
A remote run is current only after the archive stage has successfully uploaded the run record, required promoted outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
||||||
|
|
||||||
`current/run_id.txt` is the final remote commit marker and must be written last.
|
`current/run_id.txt` is the final remote commit marker and must be written last.
|
||||||
|
|
||||||
Failed, incomplete, skipped, or uncommitted archive attempts must not be presented as current remote state. Local cleanup is permitted only after successful archive commit and only when explicitly configured.
|
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured.
|
||||||
|
|
||||||
## Security and Privacy
|
## Security and Privacy
|
||||||
|
|
||||||
@@ -139,7 +139,7 @@ Rules:
|
|||||||
|
|
||||||
- Do not store raw secrets in pipeline or session YAML.
|
- Do not store raw secrets in pipeline or session YAML.
|
||||||
- Use environment variable names or secret-file references for secret handling.
|
- Use environment variable names or secret-file references for secret handling.
|
||||||
- Do not write raw secret values to manifests, logs, generated configs, or archive metadata.
|
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
|
||||||
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
|
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
|
||||||
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
|
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
|
||||||
|
|
||||||
@@ -177,7 +177,7 @@ Tests should cover:
|
|||||||
- stage success, failure, skip, and resume behavior;
|
- stage success, failure, skip, and resume behavior;
|
||||||
- adapter command construction;
|
- adapter command construction;
|
||||||
- fake storage behavior;
|
- fake storage behavior;
|
||||||
- archive commit ordering;
|
- publish commit ordering;
|
||||||
- example config validity where practical.
|
- example config validity where practical.
|
||||||
|
|
||||||
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
|
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
|
||||||
|
|||||||
285
docs/cli.md
285
docs/cli.md
@@ -6,91 +6,64 @@
|
|||||||
narratio run 2026-04-04
|
narratio run 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
This command uses default system discovery for `pipeline.yml`, the pipeline default campaign ID, and local `session.yml`. If local session discovery misses and S3 storage is configured, the positional session ID loads remote `session.yml` from the canonical session prefix.
|
This runs the full pipeline for the given session ID using default config discovery and campaign selection.
|
||||||
|
|
||||||
Default pipeline and session discovery checks system config locations only. Pass `--config`, `--campaign-file`, and `--session` to use files from the current working directory. Pass `--campaign <id>` to select a campaign from `pipeline.campaigns.root`.
|
|
||||||
|
|
||||||
Ordinary local and remote `session.yml` files must be concrete YAML. Templates belong to `narratio session init`, which renders a configured campaign template before writing the concrete file.
|
|
||||||
|
|
||||||
## Command Overview
|
## Command Overview
|
||||||
|
|
||||||
Top-level commands:
|
Top-level commands:
|
||||||
|
|
||||||
- `run <session_id>`: execute pipeline stages and persist manifest state.
|
- `run <session_id>`: execute the pipeline.
|
||||||
|
- `resume <session_id>`: continue from first non-succeeded stage.
|
||||||
- `run-stage <stage> <session_id>`: execute exactly one stage.
|
- `run-stage <stage> <session_id>`: execute exactly one stage.
|
||||||
- `resume <session_id>`: continue from first non-succeeded stage unless forced.
|
- `analyze <session_id>`: force-rerun analyze stage.
|
||||||
- `analyze <session_id>`: force-rerun the analyze stage.
|
- `publish <session_id>`: force-rerun publish stage.
|
||||||
- `publish <session_id>`: force-rerun the archive stage.
|
|
||||||
- `clean <session_id>|--all`: remove local workspace/spool state.
|
- `clean <session_id>|--all`: remove local workspace/spool state.
|
||||||
- `session <subcommand>`: session-scoped helper commands.
|
- `session <subcommand>`: session-scoped helper commands.
|
||||||
|
|
||||||
Session subcommands:
|
Session subcommands:
|
||||||
|
|
||||||
- `session init <session_id>`: create local or remote `session.yml`.
|
- `session init <session_id>`
|
||||||
- `session validate <session_id>`: run read-only preflight checks.
|
- `session plan <session_id>`
|
||||||
- `session status <session_id>`: inspect local/remote session state.
|
- `session validate <session_id>`
|
||||||
- `session plan <session_id>`: validate config, prepare workspace layout, and print stage run/skip decisions.
|
- `session status <session_id>`
|
||||||
- `session restore <session_id>`: restore durable local state from committed remote archive state.
|
- `session restore <session_id>`
|
||||||
- `session artifacts <session_id>`: list effective artifact source IDs.
|
- `session artifacts <session_id>`
|
||||||
- `session locks <session_id>`: list archive promotion locks.
|
- `session locks <session_id>`
|
||||||
- `session locks add <session_id> <source>`: add or update a remote lock.
|
- `session locks add <session_id> <source>`
|
||||||
- `session locks remove <session_id> <source>`: remove a remote lock.
|
- `session locks remove <session_id> <source>`
|
||||||
|
|
||||||
Unknown commands print usage and exit non-zero.
|
|
||||||
|
|
||||||
For config semantics, see [docs/config.md](./config.md). For operator lifecycle and recovery, see [docs/operations.md](./operations.md).
|
|
||||||
|
|
||||||
## Common Flags
|
## Common Flags
|
||||||
|
|
||||||
Most session-aware commands accept:
|
Most session-aware commands accept:
|
||||||
|
|
||||||
- `--config <path>`: optional explicit `pipeline.yml` path.
|
- `--config <pipeline.yml>`
|
||||||
- `--campaign <id>`: optional campaign ID selector.
|
- `--campaign <id>`
|
||||||
- `--campaign-file <path>`: optional explicit `campaign.yml` path.
|
- `--campaign-file <campaign.yml>`
|
||||||
- `--session <path>`: optional explicit concrete `session.yml` path.
|
- `--session <session.yml>`
|
||||||
- `--previous-session-id <value>`: expected previous session identifier.
|
- `--previous-session-id <id>`
|
||||||
|
|
||||||
The positional `<session_id>` is required even when `--session` is provided. It is used as the expected session identity and as the remote session lookup value when local session discovery misses.
|
`--campaign` and `--campaign-file` are mutually exclusive.
|
||||||
|
|
||||||
## Command Reference
|
## Command Reference
|
||||||
|
|
||||||
### `run`
|
### `run`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Runs stages in canonical order and writes manifest state.
|
||||||
- Execute configured stages in canonical order.
|
|
||||||
|
|
||||||
Success output:
|
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
|
||||||
|
|
||||||
Common failure cases:
|
|
||||||
- missing system default config/session paths when flags are omitted.
|
|
||||||
- missing selected campaign under `pipeline.campaigns.root`.
|
|
||||||
- missing local session plus missing/unavailable remote `session.yml`.
|
|
||||||
- templated `session.yml`; run `narratio session init` to generate concrete YAML.
|
|
||||||
- concrete session identity mismatch.
|
|
||||||
- unknown configured artifact key in `--artifacts`.
|
|
||||||
|
|
||||||
### `resume`
|
### `resume`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio resume <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio resume <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Starts at the first non-succeeded stage from the session manifest.
|
||||||
- Continue from session-manifest stage status.
|
|
||||||
|
|
||||||
Success output:
|
|
||||||
- `narratio resume: session <session_id> has no remaining stages`
|
|
||||||
- or `narratio resume: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
|
||||||
|
|
||||||
### `run-stage`
|
### `run-stage`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage <stage> <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--force] [--artifacts <name[,name...]>]
|
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Valid stage names:
|
Valid stage names:
|
||||||
@@ -102,175 +75,130 @@ Valid stage names:
|
|||||||
- `normalize`
|
- `normalize`
|
||||||
- `trim`
|
- `trim`
|
||||||
- `analyze`
|
- `analyze`
|
||||||
- `archive`
|
- `publish`
|
||||||
- `notify`
|
- `notify`
|
||||||
|
|
||||||
Success output:
|
`--artifacts` is accepted only for `analyze` and `publish`.
|
||||||
- `narratio run-stage: stage=<name> executed=<n> skipped=<n> force=<true|false>; manifest=<path>`
|
|
||||||
|
|
||||||
`--artifacts` is accepted only for `analyze` and `archive`.
|
|
||||||
|
|
||||||
### `analyze`
|
### `analyze`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio analyze <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
|
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Equivalent to `narratio run-stage analyze <session_id> --force`.
|
||||||
- Force-rerun the analyze stage.
|
|
||||||
- Shorter equivalent for `narratio run-stage analyze <session_id> --force`.
|
|
||||||
|
|
||||||
`analyze` is force-by-design and does not accept `--force`.
|
|
||||||
|
|
||||||
### `publish`
|
### `publish`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio publish <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--artifacts <name[,name...]>]
|
narratio publish <session_id> [--artifacts <name[,name...]>] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Equivalent to `narratio run-stage publish <session_id> --force`.
|
||||||
- Force-rerun the archive stage.
|
|
||||||
- Shorter equivalent for `narratio run-stage archive <session_id> --force`.
|
|
||||||
|
|
||||||
`publish` is force-by-design and does not accept `--force` or a stage positional argument.
|
|
||||||
|
|
||||||
### `clean`
|
### `clean`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio clean <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--clear-cache]
|
narratio clean <session_id> [--dry-run] [--clear-cache] [...common flags]
|
||||||
narratio clean --all [--config <pipeline.yml>] [--dry-run] [--clear-cache]
|
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||||
```
|
```
|
||||||
|
|
||||||
Session cleanup deletes:
|
- session mode deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
|
||||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
- `--all` deletes all session work and spool children.
|
||||||
- `{spool.root}/{campaign}/{session_id}`
|
- cache is preserved unless `--clear-cache` is passed.
|
||||||
|
|
||||||
All-session cleanup deletes:
|
|
||||||
- `{workspace.root}/work`
|
|
||||||
- the contents of `{spool.root}`, while preserving the spool root directory itself.
|
|
||||||
|
|
||||||
Cache behavior:
|
|
||||||
- cache is preserved by default.
|
|
||||||
- `--clear-cache` in session mode removes cached S3 audio files for the resolved session.
|
|
||||||
- `--all --clear-cache` removes the configured Narratio S3 audio cache namespace for the configured bucket/root prefix.
|
|
||||||
- `--clear-cache` does not delete arbitrary files under `pipeline.cache.root`.
|
|
||||||
|
|
||||||
### `session plan`
|
### `session plan`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session plan <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--force]
|
narratio session plan <session_id> [--force] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Validates config and session inputs, prepares workdir layout, and prints stage run/skip decisions.
|
||||||
- Validate config, load secrets if configured, prepare workdir, and print stage run/skip decisions.
|
|
||||||
|
|
||||||
Success output includes:
|
|
||||||
- `narratio session plan: workdir prepared at <path>`
|
|
||||||
- one line per stage (`<stage>: run|skip`)
|
|
||||||
- `totals: run=<n> skip=<n>`
|
|
||||||
|
|
||||||
### `session status`
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session status <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>]
|
|
||||||
```
|
|
||||||
|
|
||||||
Output includes:
|
|
||||||
- session ID, campaign, workspace, and session config source.
|
|
||||||
- local manifest state when present.
|
|
||||||
- remote current archive state when storage is configured.
|
|
||||||
- catalog-based promoted output availability for expected transcript and artifact sources.
|
|
||||||
- effective archive locks and conservative next actions.
|
|
||||||
|
|
||||||
### `session validate`
|
### `session validate`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session validate <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>]
|
narratio session validate <session_id> [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Checks include:
|
Read-only preflight checks for config, inputs, audio availability, previous-session requirements, publish outputs, and effective locks.
|
||||||
- effective config and session source.
|
|
||||||
- stable input files.
|
|
||||||
- local or remote audio availability.
|
|
||||||
- previous-session requirements.
|
|
||||||
- archive promotions and effective locks.
|
|
||||||
|
|
||||||
Warnings do not fail the command. Any `ERROR` finding exits non-zero.
|
### `session status`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status <session_id> [...common flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Shows local manifest state, remote current state (when storage is configured), published-output availability, and effective locks.
|
||||||
|
|
||||||
### `session init`
|
### `session init`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session init <session_id> --output ./session.yml
|
narratio session init <session_id> --output ./session.yml
|
||||||
narratio session init <session_id> --remote
|
narratio session init <session_id> --remote
|
||||||
narratio session init <session_id> --config <pipeline.yml> --campaign icewind --remote
|
narratio session init <session_id> --remote --force
|
||||||
narratio session init <session_id> --config <pipeline.yml> --campaign-file ./campaign.yml --remote
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Additional flags:
|
Flags:
|
||||||
|
|
||||||
- `--previous-session-id <value>`
|
- `--output <path>` or `--remote` (exactly one is required)
|
||||||
- `--date <value>`
|
- `--previous-session-id <id>`
|
||||||
- `--title <value>`
|
- `--date <date>`
|
||||||
- `--audio-s3-prefix <prefix>`: defaults to `audio/` when neither audio flag is provided.
|
- `--title <title>`
|
||||||
- `--audio-dir <path>`: local audio directory; mutually exclusive with `--audio-s3-prefix`.
|
- `--audio-dir <path>`
|
||||||
- `--force`: overwrite existing local or remote target.
|
- `--audio-s3-prefix <prefix>`
|
||||||
|
- `--force`
|
||||||
Behavior:
|
- common config/campaign flags
|
||||||
- exactly one of `--output` or `--remote` is required.
|
|
||||||
- `--config`, `--campaign`, and `--campaign-file` are optional overrides; omitted campaign selection uses `pipeline.campaigns.default_campaign_id`.
|
|
||||||
- `--campaign <id>` selects a campaign under `pipeline.campaigns.root`.
|
|
||||||
- `--campaign-file <path>` loads an explicit campaign file.
|
|
||||||
- if `campaign.yml` sets `session_template_file`, the template path is resolved relative to `campaign.yml` and rendered from init flags.
|
|
||||||
- if no session template is configured, a minimal concrete session file is generated directly.
|
|
||||||
- template variables must be supplied by matching flags, and supplied template-related flags must be used by the template.
|
|
||||||
- remote writes target `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
|
||||||
- existing local or remote targets fail unless `--force` is passed.
|
|
||||||
- remote writes use existence checks, not compare-and-swap.
|
|
||||||
|
|
||||||
### `session restore`
|
### `session restore`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session restore <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--dry-run] [--force] [--include-audio]
|
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Restores durable local session files from committed remote current state.
|
||||||
- Restore durable session state from the committed remote archive current state.
|
|
||||||
- Default restore installs `manifest.json`, `transcripts/**`, and `artifacts/**` from the current session archive.
|
|
||||||
- When configured previous-session inputs require it, restore reconstructs `previous/**` from the previous session's committed current archive.
|
|
||||||
- `audio/**` is restored only with `--include-audio`.
|
|
||||||
|
|
||||||
Dry-run output may include planned previous-cache downloads. Existing differing files under `previous/**` follow the normal restore conflict policy and require `--force` to overwrite.
|
Default restore scope:
|
||||||
|
|
||||||
When `--include-audio` is set, S3 audio files are restored through the shared audio cache. Cache hits avoid re-downloading large audio objects.
|
- `manifest.json`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**` when required by configured previous-session artifact inputs
|
||||||
|
|
||||||
|
`audio/**` is restored only when `--include-audio` is set.
|
||||||
|
|
||||||
### `session artifacts`
|
### `session artifacts`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session artifacts <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--remote]
|
narratio session artifacts <session_id> [--remote] [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Purpose:
|
Lists built-in sources, configured artifact sources, previous-session sources, publish output rules, and lock status. With `--remote`, includes remote published-state markers.
|
||||||
- List built-in, configured, previous-session, promoted, and locked artifact sources.
|
|
||||||
|
|
||||||
`--remote` checks promoted top-level object availability through the storage adapter. Remote markers appear only in the `Promoted` section, which reports each configured archive promotion destination and includes `dest=<path>` when that destination differs from the source's canonical path.
|
|
||||||
|
|
||||||
### `session locks`
|
### `session locks`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session locks <session_id> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>]
|
narratio session locks <session_id> [...common flags]
|
||||||
narratio session locks add <session_id> <source> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>] [--reason <text>] [--force]
|
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common flags]
|
||||||
narratio session locks remove <session_id> <source> [--config <pipeline.yml>] [--campaign <id>] [--campaign-file <campaign.yml>] [--session <session.yml>] [--previous-session-id <id>]
|
narratio session locks remove <session_id> <source> [...common flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Behavior:
|
- list mode prints effective locks from static `pipeline.publish.locks` and remote `{session_prefix}/locks.yml`.
|
||||||
- list mode prints effective locks from static `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`.
|
- add/remove mutate only the remote lock store.
|
||||||
- `locks add` writes only the remote lock store and fails if the source is already locked by pipeline config.
|
- static pipeline locks cannot be removed by lock commands.
|
||||||
- `locks remove` removes only remote locks and cannot remove static pipeline locks.
|
|
||||||
- `locks add --force` is required to update an existing remote lock reason.
|
## `--artifacts` Rules
|
||||||
|
|
||||||
|
- accepted on `run`, `resume`, `run-stage`, `analyze`, and `publish`.
|
||||||
|
- on `run-stage`, only valid for `analyze` and `publish`.
|
||||||
|
- filters configured analyze artifact execution.
|
||||||
|
- filters configured `pipeline.publish.outputs` entries for `narratio.artifact.<key>` sources.
|
||||||
|
- does not suppress built-in transcript/bounds publish outputs.
|
||||||
|
- does not imply `--force` for `run`, `resume`, or `run-stage`.
|
||||||
|
|
||||||
## Common Workflows
|
## Common Workflows
|
||||||
|
|
||||||
Default-discovery run:
|
Run full pipeline:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run 2026-04-04
|
narratio run 2026-04-04
|
||||||
@@ -282,60 +210,21 @@ Run only selected analyze artifacts:
|
|||||||
narratio run 2026-04-04 --artifacts session_recap,player_handout
|
narratio run 2026-04-04 --artifacts session_recap,player_handout
|
||||||
```
|
```
|
||||||
|
|
||||||
Resume with selected analyze artifacts:
|
Force analyze only:
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio resume 2026-04-04 --artifacts player_handout
|
|
||||||
```
|
|
||||||
|
|
||||||
Force-rerun analyze with selected artifacts:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio analyze 2026-04-04 --artifacts player_handout
|
narratio analyze 2026-04-04 --artifacts player_handout
|
||||||
```
|
```
|
||||||
|
|
||||||
Force-rerun archive publishing:
|
Force publish only:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio publish 2026-04-04
|
narratio publish 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Preview restore actions without writes:
|
Restore preview then apply:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session restore 2026-04-04 --dry-run
|
narratio session restore 2026-04-04 --dry-run
|
||||||
```
|
|
||||||
|
|
||||||
Restore and then force analyze:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session restore 2026-04-04
|
narratio session restore 2026-04-04
|
||||||
narratio analyze 2026-04-04
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Rehydrate canonical previous-session inputs after artifact-input changes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run-stage prepare 2026-04-04 --force
|
|
||||||
```
|
|
||||||
|
|
||||||
Reset local state before testing restore:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio clean 2026-04-04 --dry-run
|
|
||||||
narratio clean 2026-04-04
|
|
||||||
narratio session restore 2026-04-04 --include-audio
|
|
||||||
```
|
|
||||||
|
|
||||||
Clean all local sessions while keeping cached S3 audio:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio clean --all
|
|
||||||
```
|
|
||||||
|
|
||||||
## `--artifacts` and `--force`
|
|
||||||
|
|
||||||
- `--artifacts` filters which configured artifacts are executable when analyze runs and which configured artifact promotions archive publishes.
|
|
||||||
- `--artifacts` does not imply `--force`.
|
|
||||||
- if analyze is already `succeeded` and `--force` is not set, runner-level skip still applies.
|
|
||||||
- `--artifacts` does not suppress built-in transcript or bounds promotions.
|
|
||||||
|
|||||||
448
docs/config.md
448
docs/config.md
@@ -1,14 +1,13 @@
|
|||||||
# Configuration
|
# Configuration
|
||||||
|
|
||||||
## 1. Overview
|
## Overview
|
||||||
|
|
||||||
Narratio loads three YAML files:
|
Narratio loads three YAML files:
|
||||||
|
|
||||||
- `pipeline.yml`: pipeline-level runtime configuration.
|
- `pipeline.yml`: pipeline-level runtime settings.
|
||||||
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
|
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
|
||||||
- `session.yml`: per-session metadata and input selection, loaded locally or from the configured S3 backend.
|
- `session.yml`: per-session metadata and input selection.
|
||||||
|
|
||||||
These commands load and validate all three files before running:
|
Commands that load and validate all three files include:
|
||||||
|
|
||||||
- `narratio run`
|
- `narratio run`
|
||||||
- `narratio resume`
|
- `narratio resume`
|
||||||
@@ -23,94 +22,36 @@ These commands load and validate all three files before running:
|
|||||||
- `narratio session locks`
|
- `narratio session locks`
|
||||||
- `narratio clean <session_id>`
|
- `narratio clean <session_id>`
|
||||||
|
|
||||||
Behavior:
|
Validation behavior:
|
||||||
|
|
||||||
- strict YAML decode is enabled (`KnownFields(true)`): unknown fields fail.
|
- strict YAML decode is enabled (`KnownFields(true)`); unknown fields fail.
|
||||||
- ordinary local and remote `session.yml` files must be concrete YAML; template placeholders are rejected.
|
- loaded `session.yml` files must be concrete YAML (no `{{ ... }}` placeholders).
|
||||||
- defaults are applied for optional pipeline fields.
|
- defaults are applied for optional pipeline fields.
|
||||||
- campaign identity is selected by ID from the pipeline campaign registry unless `--campaign-file` is used.
|
- campaign/session identity mismatches fail load.
|
||||||
- campaign-level stable input paths fill missing session input paths.
|
|
||||||
- session-level stable input paths override campaign-level input paths.
|
|
||||||
- campaign config may point `session init` to a session template.
|
|
||||||
- validation enforces required fields, value formats, and cross-field constraints.
|
|
||||||
|
|
||||||
## 2. Config file discovery
|
## File Discovery
|
||||||
|
Pipeline discovery order when `--config` is omitted:
|
||||||
|
|
||||||
These commands use the same config discovery behavior:
|
|
||||||
|
|
||||||
- `narratio run`
|
|
||||||
- `narratio resume`
|
|
||||||
- `narratio run-stage`
|
|
||||||
- `narratio analyze`
|
|
||||||
- `narratio publish`
|
|
||||||
- `narratio session plan`
|
|
||||||
- `narratio session status`
|
|
||||||
- `narratio session validate`
|
|
||||||
- `narratio session restore`
|
|
||||||
- `narratio session artifacts`
|
|
||||||
- `narratio session locks`
|
|
||||||
- `narratio clean <session_id>`
|
|
||||||
|
|
||||||
Pipeline config lookup:
|
|
||||||
|
|
||||||
- if `--config <path>` is provided, that path is used.
|
|
||||||
- if omitted, Narratio searches in order:
|
|
||||||
1. `/usr/local/etc/narratio/pipeline.yml`
|
1. `/usr/local/etc/narratio/pipeline.yml`
|
||||||
2. `/etc/narratio/pipeline.yml`
|
2. `/etc/narratio/pipeline.yml`
|
||||||
- first existing file wins.
|
|
||||||
|
|
||||||
Campaign config lookup:
|
Session discovery order when `--session` is omitted:
|
||||||
|
|
||||||
- pipeline config is loaded first.
|
|
||||||
- if `--campaign-file <path>` is provided, that path is used.
|
|
||||||
- otherwise, if `--campaign <id>` is provided, Narratio loads:
|
|
||||||
- `{pipeline.campaigns.root}/{id}/campaign.yml`
|
|
||||||
- otherwise, Narratio uses `pipeline.campaigns.default_campaign_id` and loads:
|
|
||||||
- `{pipeline.campaigns.root}/{default_campaign_id}/campaign.yml`
|
|
||||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
|
||||||
- campaign IDs must be single path segments, not paths.
|
|
||||||
|
|
||||||
Session config lookup:
|
|
||||||
|
|
||||||
- if `--session <path>` is provided, that path is used.
|
|
||||||
- if `--session` is omitted, Narratio searches locally in order:
|
|
||||||
1. `/usr/local/etc/narratio/session.yml`
|
1. `/usr/local/etc/narratio/session.yml`
|
||||||
2. `/etc/narratio/session.yml`
|
2. `/etc/narratio/session.yml`
|
||||||
- first existing local file wins.
|
|
||||||
- if no local session file is found, a positional `<session_id>` is present, storage is configured, and campaign identity is resolved, Narratio loads remote `session.yml` from:
|
Campaign discovery when `--campaign-file` is omitted:
|
||||||
|
|
||||||
|
- if `--campaign <id>` is set: `{pipeline.campaigns.root}/{id}/campaign.yml`
|
||||||
|
- otherwise: `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`
|
||||||
|
|
||||||
|
Remote `session.yml` fallback:
|
||||||
|
|
||||||
|
- if local session discovery fails and storage is configured, Narratio can load:
|
||||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||||
- local discovery always runs before remote fallback.
|
|
||||||
- local files in the current working directory are used only when passed explicitly, for example `--config ./pipeline.yml --campaign-file ./campaign.yml --session ./session.yml`.
|
|
||||||
|
|
||||||
## 3. Session templating
|
## Minimal Working Config
|
||||||
|
`pipeline.yml`
|
||||||
Template behavior for local and remote `session.yml` loaded by downstream commands:
|
|
||||||
|
|
||||||
- downstream commands do not render templates.
|
|
||||||
- local and remote `session.yml` must be concrete.
|
|
||||||
- any `{{ ... }}` placeholder in loaded `session.yml` fails with guidance to run `narratio session init`.
|
|
||||||
- if concrete `session_id` mismatches the positional `<session_id>`, load fails.
|
|
||||||
- if concrete `previous_session_id` mismatches `--previous-session-id`, load fails.
|
|
||||||
|
|
||||||
Template behavior for `narratio session init`:
|
|
||||||
|
|
||||||
- `campaign.yml` may set `session_template_file`.
|
|
||||||
- relative template paths resolve relative to `campaign.yml`.
|
|
||||||
- supported init template variables:
|
|
||||||
- `{{ session_id }}`
|
|
||||||
- `{{ previous_session_id }}`
|
|
||||||
- `{{ date }}`
|
|
||||||
- `{{ title }}`
|
|
||||||
- `{{ audio_s3_prefix }}`
|
|
||||||
- `{{ audio_dir }}`
|
|
||||||
- each template variable must be supplied by the matching `session init` flag.
|
|
||||||
- template-related flags such as `--date`, `--title`, `--audio-s3-prefix`, `--audio-dir`, and `--previous-session-id` fail if the configured template does not use them.
|
|
||||||
- rendered output is strict-decoded and validated before it is written locally or remotely.
|
|
||||||
- if `session_template_file` is omitted, `session init` generates the minimal concrete session YAML directly.
|
|
||||||
|
|
||||||
## 4. Minimal config set
|
|
||||||
|
|
||||||
### `pipeline.yml`
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
campaigns:
|
campaigns:
|
||||||
@@ -120,30 +61,17 @@ whisperx:
|
|||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
```
|
```
|
||||||
|
|
||||||
Why this is sufficient:
|
`campaign.yml`
|
||||||
|
|
||||||
- `whisperx.transcribe_url` is required.
|
|
||||||
- `campaigns.default_campaign_id` selects the default campaign when `--campaign` is omitted.
|
|
||||||
- `workspace.root` defaults to `/var/lib/narratio`.
|
|
||||||
- optional sections (`seriatim`, `audita`, `archive`, `scriptorium`, `trim`, `normalize`, etc.) receive defaults or stay inactive.
|
|
||||||
|
|
||||||
### `campaign.yml`
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
campaign_id: sample-campaign
|
campaign_id: sample-campaign
|
||||||
session_template_file: ./session.template.yml
|
|
||||||
inputs:
|
inputs:
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Why this is sufficient:
|
`session.yml`
|
||||||
|
|
||||||
- `campaign_id` supplies the stable campaign identity.
|
|
||||||
- stable input files are required and resolve relative to `campaign.yml` when copied during `prepare`.
|
|
||||||
|
|
||||||
### `session.yml`
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
session_id: 2026-05-03
|
session_id: 2026-05-03
|
||||||
@@ -151,67 +79,14 @@ inputs:
|
|||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
```
|
```
|
||||||
|
|
||||||
Why this is sufficient:
|
## Publish Config
|
||||||
|
Top-level publish settings live at `pipeline.publish`.
|
||||||
- `session_id` is required.
|
|
||||||
- `campaign` can be omitted because it is supplied by `campaign.yml`.
|
|
||||||
- stable input paths can be omitted because `campaign.yml` supplies defaults.
|
|
||||||
- local `audio_dir` resolves relative to `session.yml`.
|
|
||||||
|
|
||||||
Minimal local-file usage:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run 2026-05-03 --config /path/to/pipeline.yml --campaign sample-campaign --session ./session.yml
|
|
||||||
narratio run 2026-05-03 --config /path/to/pipeline.yml --campaign-file ./campaign.yml --session ./session.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Previous-session-enabled variant:
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
session_id: 2026-05-03
|
publish:
|
||||||
previous_session_id: 2026-04-26
|
|
||||||
inputs:
|
|
||||||
audio_dir: ./audio
|
|
||||||
```
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run 2026-05-03 --config /path/to/pipeline.yml --campaign sample-campaign --session ./session.yml --previous-session-id 2026-04-26
|
|
||||||
```
|
|
||||||
|
|
||||||
## 5. Production-oriented config set
|
|
||||||
|
|
||||||
### `pipeline.yml`
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
workspace:
|
|
||||||
root: /var/lib/narratio/workspace
|
|
||||||
cleanup_after_archive: true
|
|
||||||
|
|
||||||
storage:
|
|
||||||
backend: s3
|
|
||||||
s3:
|
|
||||||
bucket: my-dnd-archive
|
|
||||||
root_prefix: dnd
|
|
||||||
region: us-east-1
|
|
||||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
|
||||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
|
||||||
|
|
||||||
campaigns:
|
|
||||||
root: /srv/narratio/campaigns
|
|
||||||
default_campaign_id: forsaken
|
|
||||||
|
|
||||||
spool:
|
|
||||||
root: /var/spool/narratio
|
|
||||||
delete_audio_after_archive: true
|
|
||||||
|
|
||||||
cache:
|
|
||||||
root: /var/cache/narratio
|
|
||||||
s3_audio: true
|
|
||||||
|
|
||||||
archive:
|
|
||||||
enabled: true
|
enabled: true
|
||||||
upload_run: true
|
upload_run: true
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
@@ -221,80 +96,31 @@ archive:
|
|||||||
locks:
|
locks:
|
||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
reason: Final recap was manually edited.
|
reason: Final recap was manually edited.
|
||||||
|
|
||||||
whisperx:
|
|
||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
|
||||||
|
|
||||||
scriptorium:
|
|
||||||
artifacts:
|
|
||||||
session_recap:
|
|
||||||
enabled: true
|
|
||||||
prompt_id: dnd.session_recap
|
|
||||||
output_path: artifacts/session_recap.md
|
|
||||||
inputs:
|
|
||||||
transcript:
|
|
||||||
source: narratio.transcript.final_trimmed
|
|
||||||
required: true
|
|
||||||
previous_recap:
|
|
||||||
source: narratio.previous_session.artifact.session_recap
|
|
||||||
required: false
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### `campaign.yml`
|
Rules:
|
||||||
|
|
||||||
```yaml
|
- `outputs[].source` is required.
|
||||||
campaign_id: forsaken
|
- `outputs[].dest` is optional; when omitted, Narratio derives destination from the source.
|
||||||
inputs:
|
- `outputs[].required` defaults to `true`.
|
||||||
speakers_file: /srv/narratio/campaigns/forsaken/speakers.yml
|
- static `publish.locks` and remote `{session_prefix}/locks.yml` are merged; static locks win on duplicates.
|
||||||
autocorrect_file: /srv/narratio/campaigns/forsaken/autocorrect.yml
|
- locks prevent overwrite of top-level published destinations.
|
||||||
glossary_file: /srv/narratio/campaigns/forsaken/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
### Local `session.yml`
|
Supported publish source families:
|
||||||
|
|
||||||
```yaml
|
- built-ins: `narratio.transcript.base`, `narratio.transcript.polished`, `narratio.transcript.final`, `narratio.transcript.final_trimmed`, `narratio.bounds.session`
|
||||||
session_id: 2026-05-03
|
- configured artifacts: `narratio.artifact.<artifact_key>`
|
||||||
previous_session_id: 2026-04-26
|
|
||||||
date: 2026-05-03
|
|
||||||
title: The Black Cabin
|
|
||||||
inputs:
|
|
||||||
audio_s3:
|
|
||||||
prefix: audio/
|
|
||||||
```
|
|
||||||
|
|
||||||
### S3-first session config
|
## Full Reference
|
||||||
|
|
||||||
For S3-first operation, upload the same `session.yml` content to:
|
|
||||||
|
|
||||||
```text
|
|
||||||
{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Then run with explicit or discovered pipeline/campaign config and no `--session`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio run 2026-05-03 --config /usr/local/etc/narratio/pipeline.yml --campaign forsaken --previous-session-id 2026-04-26
|
|
||||||
```
|
|
||||||
|
|
||||||
Operational notes:
|
|
||||||
|
|
||||||
- archive promotion is explicit and source-based via `archive.promote_artifacts`.
|
|
||||||
- `source` is required; `dest` is optional and derived when omitted.
|
|
||||||
- `archive.locks` skips top-level promotion overwrites for static locked sources while preserving run-local uploads.
|
|
||||||
- operator-created mutable locks are stored at `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml` and are merged with static locks.
|
|
||||||
- Narratio does not auto-promote all generated analyze artifacts.
|
|
||||||
- `restore` reads the same config/campaign/session inputs and restore scope is bounded by committed archive current state.
|
|
||||||
- `clean` removes workspace/spool state by default and preserves `pipeline.cache.root` unless `--clear-cache` is passed.
|
|
||||||
|
|
||||||
## 6. Full pipeline reference
|
|
||||||
|
|
||||||
|
### Pipeline
|
||||||
| Path | Type | Required | Default |
|
| Path | Type | Required | Default |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||||
| `pipeline.workspace.cleanup_after_archive` | bool | No | `false` |
|
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||||
| `pipeline.secrets.env_dir` | string | Conditional | none |
|
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||||
| `pipeline.storage.backend` | string | No | empty |
|
| `pipeline.storage.backend` | string | No | empty |
|
||||||
| `pipeline.storage.s3.bucket` | string | Conditional | empty |
|
| `pipeline.storage.s3.bucket` | string | Conditional | empty |
|
||||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||||
@@ -304,18 +130,18 @@ Operational notes:
|
|||||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||||
| `pipeline.spool.delete_audio_after_archive` | bool | No | `false` |
|
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||||
| `pipeline.archive.enabled` | bool | No | `true` |
|
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||||
| `pipeline.archive.upload_run` | bool | No | `true` |
|
| `pipeline.publish.upload_run` | bool | No | `true` |
|
||||||
| `pipeline.archive.promote_artifacts[]` | list | No | final-trimmed transcript rule |
|
| `pipeline.publish.outputs[]` | list | No | one final-trimmed output rule |
|
||||||
| `pipeline.archive.promote_artifacts[].source` | string | Yes (per rule) | none |
|
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | none |
|
||||||
| `pipeline.archive.promote_artifacts[].dest` | string | No | derived from source |
|
| `pipeline.publish.outputs[].dest` | string | No | derived from source |
|
||||||
| `pipeline.archive.promote_artifacts[].required` | bool | No | `true` |
|
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||||
| `pipeline.archive.locks[]` | list | No | empty |
|
| `pipeline.publish.locks[]` | list | No | empty |
|
||||||
| `pipeline.archive.locks[].source` | string | Yes (per lock) | none |
|
| `pipeline.publish.locks[].source` | string | Yes (per lock) | none |
|
||||||
| `pipeline.archive.locks[].reason` | string | No | empty |
|
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||||
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
|
||||||
| `pipeline.whisperx.language` | string | No | `en` |
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
|
||||||
@@ -364,162 +190,38 @@ Operational notes:
|
|||||||
| `pipeline.scriptorium.timeout` | duration string | No | `10m` |
|
| `pipeline.scriptorium.timeout` | duration string | No | `10m` |
|
||||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||||
| `pipeline.scriptorium.artifacts.<name>.enabled` | bool | No | `false` |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.depends_on[]` | list[string] | No | empty |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.render_debug` | bool | No | unset |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.prompt_id` | string | Conditional | none |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.profile_id` | string | No | empty |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.output_path` | string | Conditional | none |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.timeout` | duration string | No | empty |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` | string | Conditional | none |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.inputs.<key>.artifact` | string | No | empty |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.inputs.<key>.path` | string | No | empty |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required` | bool | No | `false` |
|
|
||||||
| `pipeline.scriptorium.artifacts.<name>.vars.<key>` | map value | No | empty |
|
|
||||||
| `pipeline.notification.backend` | string | No | empty |
|
| `pipeline.notification.backend` | string | No | empty |
|
||||||
| `pipeline.notification.recipient` | string | No | empty |
|
| `pipeline.notification.recipient` | string | No | empty |
|
||||||
| `pipeline.notification.timeout` | duration string | No | empty |
|
| `pipeline.notification.timeout` | duration string | No | `30s` |
|
||||||
|
|
||||||
Scriptorium artifact-key and dependency rules:
|
### Campaign
|
||||||
|
| Path | Type | Required |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `campaign_id` | string | Yes |
|
||||||
|
| `session_template_file` | string | No |
|
||||||
|
| `inputs.speakers_file` | string | Yes |
|
||||||
|
| `inputs.autocorrect_file` | string | Yes |
|
||||||
|
| `inputs.glossary_file` | string | Yes |
|
||||||
|
|
||||||
- artifact keys must match `^[a-z][a-z0-9_]*$`.
|
### Session
|
||||||
- enabled artifacts require `prompt_id` and `output_path`.
|
| Path | Type | Required |
|
||||||
- `output_path` must be relative, traversal-safe, and under `artifacts/`.
|
| --- | --- | --- |
|
||||||
- configured artifact input sources use `narratio.artifact.<name>`.
|
| `session_id` | string | Yes |
|
||||||
- if input source references `narratio.artifact.<name>`, artifact `<name>` must exist and must be listed in `depends_on`.
|
| `previous_session_id` | string | No |
|
||||||
- every `depends_on` entry must be a configured artifact key.
|
| `campaign` | string | No |
|
||||||
- self-dependency is rejected.
|
| `date` | string | No |
|
||||||
- enabled dependency cycles are rejected.
|
| `title` | string | No |
|
||||||
- any artifact referenced by `depends_on` or `narratio.artifact.<name>` source must define `output_path` (even if not enabled).
|
| `inputs.speakers_file` | string | No |
|
||||||
|
| `inputs.autocorrect_file` | string | No |
|
||||||
Allowed `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source` values:
|
| `inputs.glossary_file` | string | No |
|
||||||
|
| `inputs.audio_dir` | string | Conditional |
|
||||||
- `narratio.previous_session.artifact.<configured_artifact_key>`
|
| `inputs.audio_files[]` | list[string] | Conditional |
|
||||||
- `narratio.transcript.base`
|
| `inputs.audio_s3.prefix` | string | Conditional |
|
||||||
- `narratio.transcript.polished`
|
|
||||||
- `narratio.transcript.final`
|
|
||||||
- `narratio.transcript.final_trimmed`
|
|
||||||
- `narratio.bounds.session`
|
|
||||||
- `narratio.artifact.<configured_artifact_key>`
|
|
||||||
|
|
||||||
`pipeline.archive.promote_artifacts[].source` values:
|
|
||||||
|
|
||||||
- `narratio.transcript.base`
|
|
||||||
- `narratio.transcript.polished`
|
|
||||||
- `narratio.transcript.final`
|
|
||||||
- `narratio.transcript.final_trimmed`
|
|
||||||
- `narratio.bounds.session`
|
|
||||||
- `narratio.artifact.<configured_artifact_key>`
|
|
||||||
|
|
||||||
`pipeline.archive.locks[].source` accepts the same source values as `pipeline.archive.promote_artifacts[].source`.
|
|
||||||
|
|
||||||
Archive promotion destination rules:
|
|
||||||
|
|
||||||
- `dest` must be a clean relative path (not absolute, no traversal).
|
|
||||||
- duplicate `dest` values are rejected.
|
|
||||||
- if `dest` is omitted:
|
|
||||||
- built-in sources derive their canonical destination path;
|
|
||||||
- configured sources derive from `pipeline.scriptorium.artifacts.<name>.output_path`;
|
|
||||||
- derivation failure is a config validation error.
|
|
||||||
|
|
||||||
Archive lock rules:
|
|
||||||
|
|
||||||
- locks are source-based and do not accept `dest`.
|
|
||||||
- duplicate lock sources are rejected.
|
|
||||||
- static `pipeline.archive.locks` win over remote mutable locks for the same source.
|
|
||||||
- locked promotions are recorded as intentional skips in archive metadata.
|
|
||||||
- locked required promotions do not fail archive by default.
|
|
||||||
- ordinary `--force` reruns do not override locks.
|
|
||||||
|
|
||||||
Remote mutable lock store:
|
|
||||||
|
|
||||||
- path: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/locks.yml`.
|
|
||||||
- strict YAML shape: top-level `locks`, each with `source` and optional `reason`.
|
|
||||||
- `narratio session locks add` and `narratio session locks remove` mutate only the remote lock store.
|
|
||||||
- writes use existence checks plus `--force` for updates; they are not compare-and-swap atomic.
|
|
||||||
|
|
||||||
Restore-related implications:
|
|
||||||
|
|
||||||
- restore remote identity requires archive S3 identity to resolve (`pipeline.storage.s3.bucket` and session prefix derivation inputs).
|
|
||||||
- restore scope considers committed current state and durable paths (`manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`, optional `audio/**`).
|
|
||||||
- S3 audio downloads use `pipeline.spool.root` for active downloads and `pipeline.cache.root` for reusable cached audio when `pipeline.cache.s3_audio` is true.
|
|
||||||
- `pipeline.cache.root` is durable local cache state. It is not workspace state and is preserved by default by `narratio clean`.
|
|
||||||
|
|
||||||
## 7. Full campaign reference
|
|
||||||
|
|
||||||
| Path | Type | Required | Default |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `campaign.campaign_id` | string | Yes | none |
|
|
||||||
| `campaign.session_template_file` | string | No | none |
|
|
||||||
| `campaign.inputs.speakers_file` | string | Yes | none |
|
|
||||||
| `campaign.inputs.autocorrect_file` | string | Yes | none |
|
|
||||||
| `campaign.inputs.glossary_file` | string | Yes | none |
|
|
||||||
|
|
||||||
Campaign input paths and `campaign.session_template_file` may be absolute or relative. Relative paths resolve from the directory containing `campaign.yml`.
|
|
||||||
|
|
||||||
## 8. Full session reference
|
|
||||||
|
|
||||||
| Path | Type | Required | Default |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `session.session_id` | string | Yes | none |
|
|
||||||
| `session.previous_session_id` | string | No | empty |
|
|
||||||
| `session.campaign` | string | No | `campaign.campaign_id` |
|
|
||||||
| `session.date` | string | No | empty |
|
|
||||||
| `session.title` | string | No | empty |
|
|
||||||
| `session.inputs.audio_dir` | string | Conditional | empty |
|
|
||||||
| `session.inputs.audio_files[]` | list[string] | Conditional | empty |
|
|
||||||
| `session.inputs.audio_s3.prefix` | string | Conditional | none |
|
|
||||||
| `session.inputs.speakers_file` | string | No | `campaign.inputs.speakers_file` |
|
|
||||||
| `session.inputs.autocorrect_file` | string | No | `campaign.inputs.autocorrect_file` |
|
|
||||||
| `session.inputs.glossary_file` | string | No | `campaign.inputs.glossary_file` |
|
|
||||||
|
|
||||||
Session input paths may be absolute or relative. Relative audio paths and session-level stable input overrides resolve from the directory containing `session.yml`. If both `campaign.yml` and `session.yml` specify campaign identity, the values must match.
|
|
||||||
|
|
||||||
Audio-source rule:
|
|
||||||
|
|
||||||
- configure exactly one mode:
|
|
||||||
- `audio_dir`, or
|
|
||||||
- `audio_files` (at least one), or
|
|
||||||
- `audio_s3.prefix`
|
|
||||||
- `audio_s3` cannot be combined with local audio fields.
|
|
||||||
|
|
||||||
Previous-session rule:
|
|
||||||
|
|
||||||
- if `session.previous_session_id` is set, it must not equal `session.session_id`.
|
|
||||||
- canonical previous-session sources (`narratio.previous_session.artifact.<name>`) are hydrated during `prepare` from archive current state when required by enabled configured artifacts.
|
|
||||||
|
|
||||||
## 9. Secrets
|
|
||||||
|
|
||||||
Narratio supports filesystem-based secret injection via `pipeline.secrets.env_dir`.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
|
|
||||||
- `env_dir` may be absolute or relative.
|
|
||||||
- relative `env_dir` resolves from current working directory.
|
|
||||||
- files with valid env-var names (`[A-Za-z_][A-Za-z0-9_]*`) are loaded.
|
|
||||||
- values are loaded from file contents with trailing newline trimming.
|
|
||||||
- existing process env vars are preserved.
|
|
||||||
- invalid names and subdirectories are skipped.
|
|
||||||
- missing/unreadable `env_dir` fails command execution.
|
|
||||||
|
|
||||||
Guidance:
|
|
||||||
|
|
||||||
- do not put secret values directly in YAML.
|
|
||||||
- configure env var names in config and provide values via env/secrets files.
|
|
||||||
|
|
||||||
## 10. Examples
|
|
||||||
|
|
||||||
Maintained examples:
|
|
||||||
|
|
||||||
|
## Maintained Examples
|
||||||
- `examples/pipeline.minimal.yml`
|
- `examples/pipeline.minimal.yml`
|
||||||
- `examples/pipeline.production.yml`
|
- `examples/pipeline.production.yml`
|
||||||
- `examples/pipeline.full.annotated.yml`
|
- `examples/pipeline.full.annotated.yml`
|
||||||
- `examples/campaigns/sample-campaign/campaign.yml`
|
- `examples/campaigns/sample-campaign/campaign.yml`
|
||||||
- `examples/campaigns/sample-campaign/speakers.yml`
|
|
||||||
- `examples/campaigns/sample-campaign/autocorrect.yml`
|
|
||||||
- `examples/campaigns/sample-campaign/glossary.yml`
|
|
||||||
- `examples/campaigns/sample-campaign/session.template.yml`
|
|
||||||
- `examples/session.local-audio.yml`
|
- `examples/session.local-audio.yml`
|
||||||
- `examples/session.s3-audio.yml`
|
- `examples/session.s3-audio.yml`
|
||||||
|
|
||||||
These examples are validated by `internal/config` tests.
|
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ Remote-storage commands must obtain object storage through the app-level command
|
|||||||
|
|
||||||
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||||
2. Keep external transport/subprocess details in `internal/adapters`.
|
2. Keep external transport/subprocess details in `internal/adapters`.
|
||||||
3. Preserve manifest and promotion semantics expected by runner and archive logic.
|
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
|
||||||
4. Add/update stage and adapter tests.
|
4. Add/update stage and adapter tests.
|
||||||
5. Update internal component contracts in `docs/internal/`.
|
5. Update internal component contracts in `docs/internal/`.
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ Owns:
|
|||||||
- JSON output validation
|
- JSON output validation
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Transcript input selection/promotion logic (stage-owned)
|
- Transcript input selection/materialization logic (stage-owned)
|
||||||
- Bounds computation (scriptorium/trim-stage-owned)
|
- Bounds computation (scriptorium/trim-stage-owned)
|
||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ Implementation-accurate contracts for workspace/state, manifests, stages, artifa
|
|||||||
- `storage.md`: remote storage backend contracts and object-store invariants.
|
- `storage.md`: remote storage backend contracts and object-store invariants.
|
||||||
- `manifest.md`: session/run manifest schemas, lifecycle transitions, and persistence semantics.
|
- `manifest.md`: session/run manifest schemas, lifecycle transitions, and persistence semantics.
|
||||||
- `artifacts.md`: built-in artifact registry, runtime artifact catalog, and source-resolution behavior.
|
- `artifacts.md`: built-in artifact registry, runtime artifact catalog, and source-resolution behavior.
|
||||||
- `workspace.md`: local state model, manifests, run-local layout, promotion, and cleanup invariants.
|
- `workspace.md`: local state model, manifests, run-local layout, materialization, and cleanup invariants.
|
||||||
- `command-restore.md`: restore command discovery/planning/execution/reporting contract.
|
- `command-restore.md`: restore command discovery/planning/execution/reporting contract.
|
||||||
- `stage-prepare.md`: input materialization and provenance capture.
|
- `stage-prepare.md`: input materialization and provenance capture.
|
||||||
- `stage-transcribe.md`: WhisperX transcript generation.
|
- `stage-transcribe.md`: WhisperX transcript generation.
|
||||||
@@ -20,7 +20,7 @@ Implementation-accurate contracts for workspace/state, manifests, stages, artifa
|
|||||||
- `stage-normalize.md`: post-polish normalization.
|
- `stage-normalize.md`: post-polish normalization.
|
||||||
- `stage-trim.md`: bounds-driven transcript trimming.
|
- `stage-trim.md`: bounds-driven transcript trimming.
|
||||||
- `stage-analyze.md`: dependency-ordered Scriptorium artifact generation for selected configured artifacts.
|
- `stage-analyze.md`: dependency-ordered Scriptorium artifact generation for selected configured artifacts.
|
||||||
- `stage-archive.md`: archive upload and current-pointer publish contract.
|
- `stage-publish.md`: publish upload and current-pointer commit contract.
|
||||||
|
|
||||||
## External Integration Notes
|
## External Integration Notes
|
||||||
- `../integrations/README.md`: canonical location for external integration contracts (`audita.md`, `seriatim.md`, `scriptorium.md`).
|
- `../integrations/README.md`: canonical location for external integration contracts (`audita.md`, `seriatim.md`, `scriptorium.md`).
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ Default wiring and adapter calls consume:
|
|||||||
- `pipeline.seriatim.*`
|
- `pipeline.seriatim.*`
|
||||||
- `pipeline.audita.*`
|
- `pipeline.audita.*`
|
||||||
- `pipeline.scriptorium.*`
|
- `pipeline.scriptorium.*`
|
||||||
- `pipeline.storage.*` and `pipeline.archive.*` (object-store construction/gating)
|
- `pipeline.storage.*` and `pipeline.publish.*` (object-store construction/gating)
|
||||||
- `pipeline.notification.*` (sender boundary exists; placeholder behavior today)
|
- `pipeline.notification.*` (sender boundary exists; placeholder behavior today)
|
||||||
|
|
||||||
## External adapters used
|
## External adapters used
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ Inputs:
|
|||||||
Outputs:
|
Outputs:
|
||||||
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
|
||||||
- runtime catalog entries for built-ins and configured artifacts;
|
- runtime catalog entries for built-ins and configured artifacts;
|
||||||
- requirement sets for canonical previous-session inputs.
|
- requirement sets for canonical previous-session inputs;
|
||||||
- canonical S3 session, run, current, session config, session locks, audio, and promoted artifact keys.
|
- canonical S3 session, run, current, session config, session locks, audio, and published output keys.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
@@ -28,7 +28,7 @@ Owns:
|
|||||||
Does not own:
|
Does not own:
|
||||||
- prepare-stage remote hydration;
|
- prepare-stage remote hydration;
|
||||||
- stage success/skip transitions;
|
- stage success/skip transitions;
|
||||||
- archive upload orchestration.
|
- publish upload orchestration.
|
||||||
|
|
||||||
## Built-in IDs
|
## Built-in IDs
|
||||||
| Artifact ID | Canonical file | Producer stage | Output kind |
|
| Artifact ID | Canonical file | Producer stage | Output kind |
|
||||||
@@ -71,7 +71,7 @@ Previous-session canonical provenance values include:
|
|||||||
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
|
||||||
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
|
||||||
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
|
||||||
- Archive-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
|
- Publish-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
|
||||||
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
|
||||||
|
|
||||||
## Previous-session requirement scanning
|
## Previous-session requirement scanning
|
||||||
|
|||||||
@@ -27,14 +27,14 @@ Owns:
|
|||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Stage execution orchestration (`run`, `resume`, `run-stage`).
|
- Stage execution orchestration (`run`, `resume`, `run-stage`).
|
||||||
- Archive publish behavior (owned by archive stage).
|
- Publish-stage behavior.
|
||||||
- Storage transport implementation details (owned by storage adapters).
|
- Storage transport implementation details (owned by storage adapters).
|
||||||
|
|
||||||
## Config fields used
|
## Config fields used
|
||||||
- Config/session discovery and templating fields consumed by all commands.
|
- Config/session discovery and templating fields consumed by all commands.
|
||||||
- `pipeline.workspace.root` (local restore target root).
|
- `pipeline.workspace.root` (local restore target root).
|
||||||
- `pipeline.storage.*` (remote backend + archive identity derivation).
|
- `pipeline.storage.*` (remote backend + publish identity derivation).
|
||||||
- `pipeline.storage.s3.*` identity components used by archive prefix helpers.
|
- `pipeline.storage.s3.*` identity components used by session-prefix helpers.
|
||||||
- `pipeline.spool.root` for active audio downloads.
|
- `pipeline.spool.root` for active audio downloads.
|
||||||
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
|
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
@@ -80,7 +80,7 @@ Restore path scope:
|
|||||||
- Dry-run is read-only and returns plan output only.
|
- Dry-run is read-only and returns plan output only.
|
||||||
|
|
||||||
## Failure behavior
|
## Failure behavior
|
||||||
- Fails when storage backend is unavailable or archive identity cannot be resolved.
|
- Fails when storage backend is unavailable or publish identity cannot be resolved.
|
||||||
- Fails when remote current pointer/manifest is missing or invalid.
|
- Fails when remote current pointer/manifest is missing or invalid.
|
||||||
- Fails when remote manifest identity mismatches requested campaign/session.
|
- Fails when remote manifest identity mismatches requested campaign/session.
|
||||||
- Fails on local conflicts unless `--force` is set.
|
- Fails on local conflicts unless `--force` is set.
|
||||||
@@ -96,7 +96,7 @@ Restore path scope:
|
|||||||
- `internal/artifacts/archive_identity_test.go`
|
- `internal/artifacts/archive_identity_test.go`
|
||||||
|
|
||||||
## Architectural invariants
|
## Architectural invariants
|
||||||
- Restore relies on centralized archive identity/key helpers (`internal/artifacts`) rather than ad hoc key building.
|
- Restore relies on centralized path/key helpers (`internal/artifacts`) rather than ad hoc key building.
|
||||||
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
|
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
|
||||||
- Local path mapping is traversal-safe and constrained to session root.
|
- Local path mapping is traversal-safe and constrained to session root.
|
||||||
- Restore scope is deterministic and path-classified:
|
- Restore scope is deterministic and path-classified:
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ Manifest identity fields are populated by app/stage orchestration from:
|
|||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
- `pipeline.storage.s3.*` (when archive/S3 identity is set)
|
- `pipeline.storage.s3.*` (when publish/S3 identity is set)
|
||||||
|
|
||||||
## External adapters used
|
## External adapters used
|
||||||
- No external service adapters.
|
- No external service adapters.
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Stage: analyze
|
# Stage: analyze
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Execute selected configured Scriptorium artifacts in deterministic dependency order and promote successful outputs to canonical session artifact paths.
|
Execute selected configured Scriptorium artifacts in deterministic dependency order and materialize successful outputs to canonical session artifact paths.
|
||||||
|
|
||||||
## Inputs and outputs
|
## Inputs and outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
@@ -15,7 +15,7 @@ Source types used by analyze:
|
|||||||
- canonical previous-session artifacts: `narratio.previous_session.artifact.<artifact_key>`.
|
- canonical previous-session artifacts: `narratio.previous_session.artifact.<artifact_key>`.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- promoted configured artifact files at each configured `output_path`;
|
- materialized configured artifact files at each configured `output_path`;
|
||||||
- stage metadata (`generated_artifacts`, `reused_artifacts`, selected/order info).
|
- stage metadata (`generated_artifacts`, `reused_artifacts`, selected/order info).
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
@@ -24,12 +24,12 @@ Owns:
|
|||||||
- selected-artifact planning and dependency ordering;
|
- selected-artifact planning and dependency ordering;
|
||||||
- per-input resolution and required/optional handling;
|
- per-input resolution and required/optional handling;
|
||||||
- Scriptorium render/run invocation;
|
- Scriptorium render/run invocation;
|
||||||
- run-local output generation and canonical promotion.
|
- run-local output generation and canonical materialization.
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- prepare-time previous-session hydration;
|
- prepare-time previous-session hydration;
|
||||||
- object-store access for previous-session sources;
|
- object-store access for previous-session sources;
|
||||||
- archive promotion policy.
|
- publish output rule behavior.
|
||||||
|
|
||||||
## Config fields used
|
## Config fields used
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
@@ -76,5 +76,5 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural invariants
|
## Architectural invariants
|
||||||
- Canonical previous-session behavior is local-cache only during analyze.
|
- Canonical previous-session behavior is local-cache only during analyze.
|
||||||
- Generated outputs are validated and promoted before stage success is recorded.
|
- Generated outputs are validated and materialized before stage success is recorded.
|
||||||
- Resolver/catalog decisions stay deterministic and validation-gated.
|
- Resolver/catalog decisions stay deterministic and validation-gated.
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ Owns:
|
|||||||
- Per-input normalize calls to Seriatim
|
- Per-input normalize calls to Seriatim
|
||||||
- Final merge call to Seriatim
|
- Final merge call to Seriatim
|
||||||
- Run-local log/config/report path wiring
|
- Run-local log/config/report path wiring
|
||||||
- Promotion of base/report outputs to canonical paths
|
- Materialization of base/report outputs to canonical paths
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Transcript polishing or downstream artifact generation
|
- Transcript polishing or downstream artifact generation
|
||||||
@@ -43,7 +43,7 @@ Does not own:
|
|||||||
## State and Manifest Behavior
|
## State and Manifest Behavior
|
||||||
- Reads transcript inputs from transcribe stage outputs in manifest when present; falls back to canonical raw directory.
|
- Reads transcript inputs from transcribe stage outputs in manifest when present; falls back to canonical raw directory.
|
||||||
- Writes run-local outputs/logs/config under `runs/{run_id}/merge/...` when enabled.
|
- Writes run-local outputs/logs/config under `runs/{run_id}/merge/...` when enabled.
|
||||||
- Promotes canonical base transcript and optional report.
|
- Materializes canonical base transcript and optional report.
|
||||||
- Records normalized-input provenance and adapter metadata in stage metadata.
|
- Records normalized-input provenance and adapter metadata in stage metadata.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and Resume Behavior
|
||||||
@@ -59,5 +59,5 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Merge consumes normalized forms of each raw transcript.
|
- Merge consumes normalized forms of each raw transcript.
|
||||||
- Base transcript must validate before promotion.
|
- Base transcript must validate before materialization.
|
||||||
- Report output is optional and gated by config.
|
- Report output is optional and gated by config.
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ Inputs:
|
|||||||
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
|
||||||
- S3: `session.inputs.audio_s3.prefix`;
|
- S3: `session.inputs.audio_s3.prefix`;
|
||||||
- configured enabled Scriptorium artifact inputs (for previous-session requirement scanning);
|
- configured enabled Scriptorium artifact inputs (for previous-session requirement scanning);
|
||||||
- remote previous-session current archive state when previous hydration is required.
|
- remote previous-session current publish state when previous hydration is required.
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- `inputs/campaign.yml`;
|
- `inputs/campaign.yml`;
|
||||||
@@ -41,7 +41,7 @@ Owns:
|
|||||||
Does not own:
|
Does not own:
|
||||||
- transcript or artifact generation;
|
- transcript or artifact generation;
|
||||||
- analyze-stage source resolution;
|
- analyze-stage source resolution;
|
||||||
- archive commit behavior.
|
- publish commit behavior.
|
||||||
|
|
||||||
## Config fields used
|
## Config fields used
|
||||||
- `session.session_id`
|
- `session.session_id`
|
||||||
@@ -83,10 +83,10 @@ Does not own:
|
|||||||
- `narratio.previous_session.artifact.<artifact_key>`
|
- `narratio.previous_session.artifact.<artifact_key>`
|
||||||
- If one or more canonical previous-session requirements exist:
|
- If one or more canonical previous-session requirements exist:
|
||||||
- clears managed `previous/` state;
|
- clears managed `previous/` state;
|
||||||
- hydrates required/optional previous artifacts from the configured previous session’s committed archive current state;
|
- hydrates required/optional previous artifacts from the configured previous session’s committed publish current state;
|
||||||
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
|
||||||
- stores archive-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
|
- stores publish-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
|
||||||
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_archive.current`.
|
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_publish.current`.
|
||||||
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
|
||||||
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
|
||||||
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
|
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
|
||||||
@@ -95,7 +95,7 @@ Does not own:
|
|||||||
- `previous_session_id` unset:
|
- `previous_session_id` unset:
|
||||||
- if any referenced previous artifact is required: fail;
|
- if any referenced previous artifact is required: fail;
|
||||||
- if all referenced previous artifacts are optional: continue and omit them.
|
- if all referenced previous artifacts are optional: continue and omit them.
|
||||||
- Previous session archive current pointer or manifest missing:
|
- Previous session publish current pointer or manifest missing:
|
||||||
- if any referenced previous artifact is required: fail;
|
- if any referenced previous artifact is required: fail;
|
||||||
- if all referenced previous artifacts are optional: continue and omit missing ones.
|
- if all referenced previous artifacts are optional: continue and omit missing ones.
|
||||||
- Missing required previous artifact object: fail.
|
- Missing required previous artifact object: fail.
|
||||||
@@ -120,5 +120,5 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural invariants
|
## Architectural invariants
|
||||||
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
|
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
|
||||||
- Storage keys are computed by callers using archive/path helpers; storage adapter receives explicit keys.
|
- Storage keys are computed by callers using path helpers; storage adapter receives explicit keys.
|
||||||
- `prepare` is the only stage that hydrates canonical previous-session cache state.
|
- `prepare` is the only stage that hydrates canonical previous-session cache state.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Stage: archive
|
# Stage: publish
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Publish durable run/session state to object storage, then atomically advance remote current state.
|
Publish durable run/session state to object storage, then atomically advance remote current state.
|
||||||
@@ -7,37 +7,37 @@ Publish durable run/session state to object storage, then atomically advance rem
|
|||||||
Inputs:
|
Inputs:
|
||||||
- session manifest and prerequisite stage records
|
- session manifest and prerequisite stage records
|
||||||
- run root contents under `runs/{run_id}/`
|
- run root contents under `runs/{run_id}/`
|
||||||
- promotion rules with artifact `source` IDs and archive `dest` paths (`archive.promote_artifacts`)
|
- publish output rules with artifact `source` IDs and publish `dest` paths (`pipeline.publish.outputs`)
|
||||||
- effective source-based promotion locks from static config and remote session lock store
|
- effective source-based publish locks from static config and remote session lock store
|
||||||
- session-level `previous/**` cache files when present
|
- session-level `previous/**` cache files when present
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
|
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
|
||||||
- uploaded promoted artifacts under `{session_prefix}/...`
|
- uploaded published outputs under `{session_prefix}/...`
|
||||||
- uploaded session previous-cache files under `{session_prefix}/previous/...` when present
|
- uploaded session previous-cache files under `{session_prefix}/previous/...` when present
|
||||||
- `{session_prefix}/current/manifest.json`
|
- `{session_prefix}/current/manifest.json`
|
||||||
- `{session_prefix}/current/run_id.txt` written last
|
- `{session_prefix}/current/run_id.txt` written last
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
Owns:
|
Owns:
|
||||||
- Archive enable/disable gate behavior
|
- publish enable/disable gate behavior
|
||||||
- Prerequisite stage success enforcement
|
- prerequisite stage success enforcement
|
||||||
- Run file collection and upload (excluding `audio/`)
|
- run file collection and upload (excluding `audio/`)
|
||||||
- Promotion rule resolution and upload
|
- publish output rule resolution and upload
|
||||||
- Promotion lock enforcement
|
- publish lock enforcement
|
||||||
- Session previous-cache file collection/upload
|
- session previous-cache file collection/upload
|
||||||
- Commit pointer publish order
|
- commit pointer publish order
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Stage execution before archive
|
- stage execution before publish
|
||||||
- Post-archive local cleanup policy execution (handled by app cleanup logic)
|
- post-publish local cleanup policy execution (handled by app cleanup logic)
|
||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
- `pipeline.archive.enabled`
|
- `pipeline.publish.enabled`
|
||||||
- `pipeline.archive.upload_run`
|
- `pipeline.publish.upload_run`
|
||||||
- `pipeline.archive.promote_artifacts`
|
- `pipeline.publish.outputs`
|
||||||
- `pipeline.archive.locks`
|
- `pipeline.publish.locks`
|
||||||
- `{session_prefix}/locks.yml` loaded by app orchestration before archive execution
|
- `{session_prefix}/locks.yml` loaded by app orchestration before publish execution
|
||||||
- `pipeline.storage.s3.bucket`
|
- `pipeline.storage.s3.bucket`
|
||||||
- `pipeline.storage.s3.root_prefix`
|
- `pipeline.storage.s3.root_prefix`
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
@@ -51,26 +51,28 @@ Does not own:
|
|||||||
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
|
||||||
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
- Resolves bucket/prefix from manifest identity first, then config fallback.
|
||||||
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
|
||||||
- Skips top-level promotion uploads for effective locked sources; run-local uploads still publish.
|
- Skips top-level published output uploads for effective locked sources; run-local materialized outputs remain unchanged.
|
||||||
- When selected configured artifact keys are supplied, skips promotion rules for unselected `narratio.artifact.<key>` sources; built-in transcript and bounds promotions still publish.
|
- When selected configured artifact keys are supplied, skips publish rules for unselected `narratio.artifact.<key>` sources; built-in transcript and bounds outputs still publish.
|
||||||
- Effective locks are the union of `pipeline.archive.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
|
- Effective locks are the union of `pipeline.publish.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
|
||||||
- Writes metadata including:
|
- Writes metadata including:
|
||||||
- upload counts/paths
|
- upload counts/paths
|
||||||
- `previous_files_uploaded` and `previous_uploaded_paths`
|
- `previous_files_uploaded` and `previous_uploaded_paths`
|
||||||
- `skipped_unselected_promotions`
|
- `published_files_uploaded` and `published_paths`
|
||||||
- `locked_promotion_count` and `locked_promotions`
|
- `skipped_optional_outputs`
|
||||||
|
- `skipped_unselected_outputs`
|
||||||
|
- `locked_output_count` and `locked_outputs`
|
||||||
- `current_manifest_key`
|
- `current_manifest_key`
|
||||||
- `current_run_id_key`
|
- `current_run_id_key`
|
||||||
- `current_pointer_written`
|
- `current_pointer_written`
|
||||||
- On skipped archive path, returns metadata with `skipped=true` and pointer not written.
|
- On skipped publish path, returns metadata with `skipped=true` and pointer not written.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and Resume Behavior
|
||||||
- Stage may self-skip (metadata skip) when archive disabled or run upload disabled.
|
- Stage may self-skip (metadata skip) when publish disabled or run upload disabled.
|
||||||
- Runner-level skip also applies for previously succeeded stage unless forced.
|
- Runner-level skip also applies for previously succeeded stage unless forced.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required promotion source, upload failures, or pointer write failures.
|
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required output source, upload failures, or pointer write failures.
|
||||||
- Locked required promotions are intentional skips and do not fail archive.
|
- Locked required outputs are intentional skips and do not fail publish.
|
||||||
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Tests to Inspect Before Changing
|
||||||
@@ -79,8 +81,8 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Run upload excludes `audio/` subtree.
|
- Run upload excludes `audio/` subtree.
|
||||||
- Session `previous/**` is archiveable durable input/provenance state, not run-local output.
|
- Session `previous/**` is publishable durable input/provenance state, not run-local output.
|
||||||
- Ordinary `--force` does not override archive locks.
|
- Ordinary `--force` does not override publish locks.
|
||||||
- Malformed or unreadable remote lock store fails archive-capable execution before promotion.
|
- Malformed or unreadable remote lock store fails publish-capable execution before output uploads.
|
||||||
- `current/manifest.json` uploads before `current/run_id.txt`.
|
- `current/manifest.json` uploads before `current/run_id.txt`.
|
||||||
- `current/run_id.txt` is the remote publish commit marker.
|
- `current/run_id.txt` is the remote publish commit marker.
|
||||||
@@ -15,7 +15,7 @@ Owns:
|
|||||||
- Discovering prepared audio inputs
|
- Discovering prepared audio inputs
|
||||||
- Deriving speaker ids from audio basenames
|
- Deriving speaker ids from audio basenames
|
||||||
- Parallel WhisperX invocation with bounded concurrency
|
- Parallel WhisperX invocation with bounded concurrency
|
||||||
- Validating produced JSON and promoting run-local outputs
|
- Validating produced JSON and materializing run-local outputs
|
||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Transcript merge/polish/normalize/trim/analyze
|
- Transcript merge/polish/normalize/trim/analyze
|
||||||
@@ -36,8 +36,8 @@ Does not own:
|
|||||||
|
|
||||||
## State and Manifest Behavior
|
## State and Manifest Behavior
|
||||||
- Uses run-local output paths under `runs/{run_id}/transcribe/outputs/...` when run layout is enabled.
|
- Uses run-local output paths under `runs/{run_id}/transcribe/outputs/...` when run layout is enabled.
|
||||||
- Validates each generated transcript JSON before promotion.
|
- Validates each generated transcript JSON before materialization.
|
||||||
- Promotes canonical outputs to `transcripts/raw/*.json`.
|
- Materializes canonical outputs to `transcripts/raw/*.json`.
|
||||||
- Records per-file metadata (attempts/status/duration/output path) in stage metadata.
|
- Records per-file metadata (attempts/status/duration/output path) in stage metadata.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and Resume Behavior
|
||||||
@@ -54,5 +54,5 @@ Does not own:
|
|||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Speaker identity is derived from `.flac` basename and must be unique.
|
- Speaker identity is derived from `.flac` basename and must be unique.
|
||||||
- Every successful speaker output must be valid JSON before promotion.
|
- Every successful speaker output must be valid JSON before materialization.
|
||||||
- Canonical raw transcript set is the only supported merge input surface.
|
- Canonical raw transcript set is the only supported merge input surface.
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ Does not own:
|
|||||||
## State and Manifest Behavior
|
## State and Manifest Behavior
|
||||||
- Reads final transcript from normalize manifest outputs when available; falls back to canonical path.
|
- Reads final transcript from normalize manifest outputs when available; falls back to canonical path.
|
||||||
- Uses run-local outputs/logs/reports/config/scratch paths when run layout is enabled.
|
- Uses run-local outputs/logs/reports/config/scratch paths when run layout is enabled.
|
||||||
- Promotes canonical final-trimmed transcript; promotes session bounds when trim enabled.
|
- Materializes canonical final-trimmed transcript and session bounds when trim is enabled.
|
||||||
- Records bounds diagnostics, trim action, keep selector, and adapter metadata.
|
- Records bounds diagnostics, trim action, keep selector, and adapter metadata.
|
||||||
|
|
||||||
## Skip and Resume Behavior
|
## Skip and Resume Behavior
|
||||||
|
|||||||
@@ -37,12 +37,12 @@ Does not own:
|
|||||||
## External adapters used
|
## External adapters used
|
||||||
Storage package contracts:
|
Storage package contracts:
|
||||||
- `ObjectStore` (active remote object-store boundary): `List`, `Download`, `Upload`, `Exists`.
|
- `ObjectStore` (active remote object-store boundary): `List`, `Download`, `Upload`, `Exists`.
|
||||||
- `Backend` (archive request boundary): currently implemented with `NoopBackend` only.
|
- `Backend` (legacy compatibility boundary): currently implemented with `NoopBackend` only.
|
||||||
|
|
||||||
Implementations:
|
Implementations:
|
||||||
- `S3Backend`: AWS SDK-backed `ObjectStore` implementation.
|
- `S3Backend`: AWS SDK-backed `ObjectStore` implementation.
|
||||||
- `FakeBackend`: deterministic test `ObjectStore` and archive backend.
|
- `FakeBackend`: deterministic test `ObjectStore` and compatibility backend.
|
||||||
- `NoopBackend`: deterministic no-op archive backend for compatibility wiring.
|
- `NoopBackend`: deterministic no-op compatibility backend for wiring/tests.
|
||||||
|
|
||||||
## State and manifest behavior
|
## State and manifest behavior
|
||||||
- Storage implementations are stateless with respect to manifest/session lifecycle.
|
- Storage implementations are stateless with respect to manifest/session lifecycle.
|
||||||
@@ -67,7 +67,7 @@ Implementations:
|
|||||||
- `internal/adapters/storage/s3_backend_test.go`
|
- `internal/adapters/storage/s3_backend_test.go`
|
||||||
- `internal/adapters/storage/fake_test.go`
|
- `internal/adapters/storage/fake_test.go`
|
||||||
- `internal/adapters/storage/keys_test.go`
|
- `internal/adapters/storage/keys_test.go`
|
||||||
- `internal/adapters/storage/archive.go` + consumers in stage tests (`prepare`, `archive`)
|
- `internal/adapters/storage/archive.go` + consumers in stage tests (`prepare`, `publish`)
|
||||||
|
|
||||||
## Architectural invariants
|
## Architectural invariants
|
||||||
- Callers pass full bucket-relative keys.
|
- Callers pass full bucket-relative keys.
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Workspace internals
|
# Workspace internals
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Define the local durable and run-local workspace model used by stages, manifests, resume, and archive.
|
Define the local durable and run-local workspace model used by stages, manifests, resume, and publish.
|
||||||
|
|
||||||
## Inputs and Outputs
|
## Inputs and Outputs
|
||||||
Inputs:
|
Inputs:
|
||||||
@@ -24,14 +24,14 @@ Owns:
|
|||||||
|
|
||||||
Does not own:
|
Does not own:
|
||||||
- Stage business logic
|
- Stage business logic
|
||||||
- Remote archive semantics (documented in `stage-archive.md`)
|
- Remote publish semantics (documented in `stage-publish.md`)
|
||||||
- CLI argument parsing
|
- CLI argument parsing
|
||||||
|
|
||||||
## Config Fields Used
|
## Config Fields Used
|
||||||
- `pipeline.workspace.root`
|
- `pipeline.workspace.root`
|
||||||
- `pipeline.workspace.cleanup_after_archive`
|
- `pipeline.workspace.cleanup_after_publish`
|
||||||
- `pipeline.spool.root`
|
- `pipeline.spool.root`
|
||||||
- `pipeline.spool.delete_audio_after_archive`
|
- `pipeline.spool.delete_audio_after_publish`
|
||||||
- `pipeline.cache.root`
|
- `pipeline.cache.root`
|
||||||
- `pipeline.cache.s3_audio`
|
- `pipeline.cache.s3_audio`
|
||||||
- `session.campaign`
|
- `session.campaign`
|
||||||
@@ -43,10 +43,10 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
## State and Manifest Behavior
|
## State and Manifest Behavior
|
||||||
- Session state is persisted in the session manifest (`manifest.Manifest`).
|
- Session state is persisted in the session manifest (`manifest.Manifest`).
|
||||||
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
|
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
|
||||||
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and promoted to canonical session paths after stage success.
|
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and then materialized to canonical session paths after stage success.
|
||||||
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
|
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
|
||||||
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
|
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
|
||||||
- `previous/**` is reconstructed from configured previous-session requirements; restore uses the previous session's committed current archive rather than treating current-session archived `previous/**` as authoritative.
|
- `previous/**` is reconstructed from configured previous-session requirements; restore uses the previous session's committed current publish state rather than treating current-session stored `previous/**` as authoritative.
|
||||||
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
|
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
|
||||||
- `narratio clean <id>` removes the session work root and session spool root.
|
- `narratio clean <id>` removes the session work root and session spool root.
|
||||||
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
|
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
|
||||||
@@ -60,7 +60,7 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
- Failures preserve manifests and run-local files for inspection.
|
- Failures preserve manifests and run-local files for inspection.
|
||||||
- Lock conflicts fail fast via `ErrLockConflict`.
|
- Lock conflicts fail fast via `ErrLockConflict`.
|
||||||
- Cleanup can fail post-archive; failure is recorded in archive stage metadata and returned by the run.
|
- Cleanup can fail post-publish; failure is recorded in publish stage metadata and returned by the run.
|
||||||
|
|
||||||
## Tests to Inspect Before Changing
|
## Tests to Inspect Before Changing
|
||||||
- `internal/artifacts/local_test.go`
|
- `internal/artifacts/local_test.go`
|
||||||
@@ -72,7 +72,7 @@ None directly in this subsystem. Stages may use object storage adapters and then
|
|||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
|
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
|
||||||
- Run roots are always nested: `runs/{run_id}` under the session root.
|
- Run roots are always nested: `runs/{run_id}` under the session root.
|
||||||
- Run-local output promotion must end in canonical session paths.
|
- Run-local output materialization must end in canonical session paths.
|
||||||
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
|
||||||
- Automatic post-archive cleanup only targets run-scoped directories and must never delete configured root directories.
|
- Automatic post-publish cleanup only targets run-scoped directories and must never delete configured root directories.
|
||||||
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.
|
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.
|
||||||
|
|||||||
@@ -1,285 +1,170 @@
|
|||||||
# Operations
|
# Operations
|
||||||
|
|
||||||
This guide describes the implemented operator lifecycle for Narratio.
|
This guide covers the implemented operator lifecycle for Narratio.
|
||||||
|
|
||||||
For field-level configuration, see [docs/config.md](./config.md). For full command/flag reference, see [docs/cli.md](./cli.md).
|
For field-level settings, see [docs/config.md](./config.md). For syntax/flags, see [docs/cli.md](./cli.md).
|
||||||
|
|
||||||
## Normal workflow (S3-first path)
|
## Normal Workflow
|
||||||
|
|
||||||
1. Create or upload `session.yml`, or pass a local `session.yml` explicitly.
|
1. Ensure `pipeline.yml`, `campaign.yml`, and `session.yml` are available.
|
||||||
2. Upload session `.flac` files to object storage under the configured session audio prefix.
|
2. Ensure session audio is available (local `audio_dir`/`audio_files` or S3 prefix).
|
||||||
3. Run Narratio:
|
3. Run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run 2026-04-04
|
narratio run 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
4. Read success output:
|
4. Inspect status:
|
||||||
- `narratio run: session <session_id>; executed=<n> skipped=<n>; manifest=<path>`
|
|
||||||
- use `narratio session status <session_id>` for inspection.
|
|
||||||
|
|
||||||
Notes:
|
|
||||||
- default pipeline/session discovery checks system config locations; campaign selection uses `pipeline.campaigns.default_campaign_id` unless `--campaign <id>` or `--campaign-file <path>` is passed.
|
|
||||||
- when local `session.yml` discovery misses, positional `<session_id>` loads remote `session.yml` from `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`.
|
|
||||||
- S3 audio mode requires `session.inputs.audio_s3.prefix` and valid object-store access.
|
|
||||||
|
|
||||||
Initialize a remote session skeleton:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session init 2026-04-04 --remote
|
|
||||||
```
|
|
||||||
|
|
||||||
Remote init uses normal default config discovery and writes `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`. If `campaign.yml` sets `session_template_file`, init renders that template from the supplied flags and writes concrete YAML. Pass `--config`, `--campaign <id>`, or `--campaign-file <path>` when testing non-system config files. It fails if the object already exists unless `--force` is passed.
|
|
||||||
|
|
||||||
Validate before running:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session validate 2026-04-04
|
|
||||||
```
|
|
||||||
|
|
||||||
## Restore workflow
|
|
||||||
|
|
||||||
Use restore when local durable session state is missing or stale and archive current state is authoritative.
|
|
||||||
|
|
||||||
Dry-run (no local writes):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session restore 2026-04-04 --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
Execution:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session restore 2026-04-04
|
|
||||||
```
|
|
||||||
|
|
||||||
Post-restore analyze rerun pattern:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio analyze 2026-04-04
|
|
||||||
```
|
|
||||||
|
|
||||||
Restore source-of-truth:
|
|
||||||
- remote commit marker: `current/run_id.txt`
|
|
||||||
- remote current manifest: `current/manifest.json`
|
|
||||||
- configured previous-session requirements are reconstructed from the previous session's remote `current/` state, not from archived `previous/**` objects in the current session.
|
|
||||||
|
|
||||||
Restore default scope:
|
|
||||||
- includes `manifest.json`, `transcripts/**`, and `artifacts/**` from the current session archive.
|
|
||||||
- includes `previous/**` only when configured previous-session artifact inputs require it; restore hydrates those files the same way `prepare` would.
|
|
||||||
- includes `audio/**` only with `--include-audio`
|
|
||||||
- excludes `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`, and `current/**` (except remote `current/manifest.json` as source)
|
|
||||||
|
|
||||||
Reset local state before restore testing:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio clean 2026-04-04 --dry-run
|
|
||||||
narratio clean 2026-04-04
|
|
||||||
narratio session restore 2026-04-04 --include-audio
|
|
||||||
```
|
|
||||||
|
|
||||||
`clean` removes the local session work directory and session spool directory. It preserves the durable S3 audio cache by default, so repeated restore or forced prepare tests do not re-download large audio files.
|
|
||||||
|
|
||||||
## Local filesystem layout and state artifacts
|
|
||||||
|
|
||||||
Session root:
|
|
||||||
- `{workspace.root}/work/{campaign}/{session_id}/`
|
|
||||||
|
|
||||||
Primary state:
|
|
||||||
- `manifest.json`: session-level stage state.
|
|
||||||
- `runs/{run_id}/manifest.json`: invocation-level state.
|
|
||||||
- `.lock`: session lock while a modifying command is active.
|
|
||||||
- `inputs/campaign.yml`, `inputs/session.yml`, and `inputs/pipeline.resolved.yml`: materialized config inputs for the run.
|
|
||||||
|
|
||||||
Canonical session directories:
|
|
||||||
- `inputs/`
|
|
||||||
- `audio/`
|
|
||||||
- `transcripts/`
|
|
||||||
- `artifacts/`
|
|
||||||
- `previous/`
|
|
||||||
- `reports/`
|
|
||||||
- `logs/`
|
|
||||||
- `config/`
|
|
||||||
- `current/`
|
|
||||||
- `runs/`
|
|
||||||
|
|
||||||
Run-local stage directories:
|
|
||||||
- `runs/{run_id}/{stage}/` with stage-local `outputs/`, `logs/`, `reports/`, `config/`, `scratch/`.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
- directory creation is idempotent.
|
|
||||||
- stage outputs are generally generated run-local first, then promoted to canonical paths on success.
|
|
||||||
- restore installs downloaded files to canonical session paths and does not recreate historical run sandboxes.
|
|
||||||
|
|
||||||
## Analyze artifact execution lifecycle
|
|
||||||
|
|
||||||
Analyze executes configured artifacts from `pipeline.scriptorium.artifacts`.
|
|
||||||
|
|
||||||
Execution model:
|
|
||||||
- executable set = enabled artifacts, filtered by `--artifacts` when provided.
|
|
||||||
- artifact-to-artifact dependencies are declared via `depends_on`.
|
|
||||||
- selected artifacts run in deterministic dependency order.
|
|
||||||
- after each successful artifact run, output is promoted to configured canonical `output_path`.
|
|
||||||
|
|
||||||
Configured artifact source reuse:
|
|
||||||
- a non-executable configured artifact can satisfy inputs if its configured output file already exists and is valid.
|
|
||||||
- reused configured artifact provenance is `filesystem.disabled_artifact_output`.
|
|
||||||
|
|
||||||
`--artifacts` behavior:
|
|
||||||
- accepted on `run`, `resume`, `run-stage analyze`, `run-stage archive`, `analyze`, and `publish`.
|
|
||||||
- filters analyze execution and configured artifact promotions.
|
|
||||||
- built-in transcript and bounds promotions are not filtered.
|
|
||||||
- does not imply force on `run`, `resume`, or `run-stage`; `narratio analyze` is force-by-design.
|
|
||||||
- `publish` is force-by-design and accepts `--artifacts` for configured artifact promotions.
|
|
||||||
|
|
||||||
Canonical previous-session input behavior:
|
|
||||||
- canonical sources use `narratio.previous_session.artifact.<artifact_key>`.
|
|
||||||
- these inputs are hydrated by `prepare` and by `restore`; `analyze` expects the local previous cache to already exist.
|
|
||||||
- if analyze fails due to missing canonical previous cache, rerun:
|
|
||||||
- `narratio run-stage prepare <id> --force`
|
|
||||||
- or `narratio session restore <id>` when remote archive current state is authoritative.
|
|
||||||
|
|
||||||
## Remote archive layout and publish contract
|
|
||||||
|
|
||||||
Preferred manual publish command:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio publish <id>
|
|
||||||
```
|
|
||||||
|
|
||||||
`publish` is equivalent to `narratio run-stage archive <id> --force`; use `run-stage` when you need the general single-stage command form.
|
|
||||||
|
|
||||||
When archive is enabled and run upload is enabled, archive publishes under:
|
|
||||||
|
|
||||||
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
|
||||||
- run prefix: `{session_prefix}/runs/{run_id}/`
|
|
||||||
|
|
||||||
Archive uploads:
|
|
||||||
- run record files from run root (excluding `audio/`).
|
|
||||||
- promoted files from explicit `archive.promote_artifacts` rules.
|
|
||||||
- mutable session locks from helper commands live at `{session_prefix}/locks.yml`.
|
|
||||||
|
|
||||||
Publish order:
|
|
||||||
1. upload `current/manifest.json`
|
|
||||||
2. upload `current/run_id.txt` last
|
|
||||||
|
|
||||||
`current/run_id.txt` is the remote commit marker.
|
|
||||||
|
|
||||||
Archive promotion is explicit and source-based:
|
|
||||||
- Narratio does not auto-promote all generated analyze artifacts.
|
|
||||||
- each rule resolves `source` through the artifact resolver/catalog model, then uploads to `dest`.
|
|
||||||
- missing required promotion sources fail archive stage.
|
|
||||||
- missing optional promotion sources are skipped.
|
|
||||||
- invalid resolved artifacts fail archive stage.
|
|
||||||
- `archive.locks` skips top-level promotion overwrites for locked sources while run-local uploads still publish.
|
|
||||||
- remote locks from `{session_prefix}/locks.yml` are merged with static `archive.locks`; static locks win on duplicate sources.
|
|
||||||
- locked required promotions are treated as intentional successful skips and are recorded in archive metadata.
|
|
||||||
|
|
||||||
Lock helper behavior:
|
|
||||||
- `narratio session locks <id>` lists effective static and remote locks.
|
|
||||||
- `narratio session locks add <id> <source> --reason <text>` writes a remote lock.
|
|
||||||
- `narratio session locks add <id> <source> --force --reason <text>` updates an existing remote lock reason.
|
|
||||||
- `narratio session locks remove <id> <source>` removes only a remote lock.
|
|
||||||
- `locks remove` cannot remove static pipeline locks.
|
|
||||||
- remote lock writes check whether the lock store exists, but are not compare-and-swap atomic.
|
|
||||||
|
|
||||||
## Resume, retry, restore, and safe rerun behavior
|
|
||||||
|
|
||||||
Default skip:
|
|
||||||
- `run` and `run-stage` skip already-succeeded stages unless `--force` is set.
|
|
||||||
|
|
||||||
Resume:
|
|
||||||
- `resume` starts at first non-succeeded stage.
|
|
||||||
- `resume --force` runs full stage order.
|
|
||||||
|
|
||||||
Restore conflict policy:
|
|
||||||
- restore classifies local differences as conflicts.
|
|
||||||
- without `--force`, restore fails when conflicts exist.
|
|
||||||
- with `--force`, conflicting local files are overwritten by remote archive files.
|
|
||||||
|
|
||||||
Forced reruns:
|
|
||||||
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
|
||||||
- ordinary `--force` does not override archive locks.
|
|
||||||
|
|
||||||
Safe rerun pattern:
|
|
||||||
1. rerun the changed stage with `--force`.
|
|
||||||
2. run `resume` to rebuild downstream stages.
|
|
||||||
|
|
||||||
## Cleanup behavior
|
|
||||||
|
|
||||||
Automatic post-archive cleanup is considered only when archive stage executed and succeeded.
|
|
||||||
|
|
||||||
Automatic cleanup toggles:
|
|
||||||
- `pipeline.spool.delete_audio_after_archive=true` deletes run-scoped spool audio.
|
|
||||||
- `pipeline.workspace.cleanup_after_archive=true` deletes run-scoped local run directory.
|
|
||||||
|
|
||||||
Manual cleanup:
|
|
||||||
- `narratio clean <id>` deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
|
|
||||||
- `narratio clean --all` deletes all local session work under `{workspace.root}/work` and all spool children under `{spool.root}`.
|
|
||||||
- `--dry-run` prints targets without deleting.
|
|
||||||
- `--clear-cache` also removes matching S3 audio cache files. Without it, cache is preserved.
|
|
||||||
|
|
||||||
The S3 audio cache under `pipeline.cache.root` is durable input cache state, not workspace or spool state. Automatic cleanup and default manual cleanup do not delete it.
|
|
||||||
|
|
||||||
Cleanup eligibility gates:
|
|
||||||
- archive enabled
|
|
||||||
- archive run upload enabled
|
|
||||||
- run record upload completed
|
|
||||||
- current pointer write completed (`current/run_id.txt` written)
|
|
||||||
|
|
||||||
No cleanup for failed/incomplete/unarchived/archive-skipped runs.
|
|
||||||
|
|
||||||
## Failure and recovery playbooks
|
|
||||||
|
|
||||||
After run failure, Narratio keeps:
|
|
||||||
- session manifest
|
|
||||||
- run manifest
|
|
||||||
- run-local artifacts/logs/config/reports
|
|
||||||
|
|
||||||
Failed or incomplete runs remain local-only.
|
|
||||||
|
|
||||||
After restore failure:
|
|
||||||
- already-installed restore files remain in place.
|
|
||||||
- restore does not roll back prior successful installs.
|
|
||||||
- existing local manifest is preserved if restored manifest validation/install fails.
|
|
||||||
|
|
||||||
Recommended recovery:
|
|
||||||
|
|
||||||
1. inspect state:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session status 2026-04-04
|
narratio session status 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
This reports local manifest state, committed remote current state, expected remote transcript/artifact availability, and archive locks.
|
## Publish Workflow
|
||||||
|
|
||||||
2. for restore-specific checks, run:
|
Publish is the stage that commits remote current state.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio publish 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalent command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage publish 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish uploads:
|
||||||
|
|
||||||
|
- run history files under `{session_prefix}/runs/{run_id}/` (excluding `audio/`)
|
||||||
|
- configured published outputs from `pipeline.publish.outputs`
|
||||||
|
- `previous/**` cache files when present
|
||||||
|
- `current/manifest.json`
|
||||||
|
- `current/run_id.txt` last
|
||||||
|
|
||||||
|
`current/run_id.txt` is the remote commit marker.
|
||||||
|
|
||||||
|
## Published Outputs and Locks
|
||||||
|
|
||||||
|
Published output behavior:
|
||||||
|
|
||||||
|
- outputs are source-based rules in `pipeline.publish.outputs`.
|
||||||
|
- required missing unlocked sources fail publish.
|
||||||
|
- optional missing unlocked sources are skipped.
|
||||||
|
- selected artifacts (`--artifacts`) only filter configured `narratio.artifact.<key>` output rules.
|
||||||
|
- built-in transcript and bounds output rules are not filtered by `--artifacts`.
|
||||||
|
|
||||||
|
Lock behavior:
|
||||||
|
|
||||||
|
- static locks: `pipeline.publish.locks`.
|
||||||
|
- mutable locks: `{session_prefix}/locks.yml`.
|
||||||
|
- effective lock set is static + mutable; static wins on duplicate sources.
|
||||||
|
- locked outputs are intentional skips and do not fail publish.
|
||||||
|
- lock commands mutate only remote mutable locks.
|
||||||
|
|
||||||
|
## Restore Workflow
|
||||||
|
|
||||||
|
Use restore when local durable session state is missing/stale and committed remote current state is authoritative.
|
||||||
|
|
||||||
|
Preview:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session restore 2026-04-04 --dry-run
|
narratio session restore 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
3. fix root cause (config/input/credentials/storage/service availability).
|
Apply:
|
||||||
4. continue with `resume`, or targeted `run-stage <stage> <id> --force` followed by `resume`.
|
|
||||||
|
|
||||||
## Restore report
|
```bash
|
||||||
|
narratio session restore 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
Non-dry-run restore writes a durable report at:
|
Default restore scope:
|
||||||
- `reports/restore-latest.json`
|
|
||||||
|
|
||||||
Report content includes:
|
- `manifest.json`
|
||||||
- identity (`campaign`, `session_id`, `run_id`)
|
- `transcripts/**`
|
||||||
- mode flags (`dry_run`, `force`, `include_audio`)
|
- `artifacts/**`
|
||||||
- plan counts and execution counts
|
- `previous/**` when required by configured previous-session artifact inputs
|
||||||
- per-action status
|
|
||||||
|
|
||||||
Dry-run does not write restore report files.
|
Optional:
|
||||||
|
|
||||||
## Operational caveats
|
- add `--include-audio` to restore `audio/**`.
|
||||||
|
|
||||||
- `session status <session_id>` uses normal config/session loading, including remote session fallback.
|
Restore reads committed current state only (`current/run_id.txt`, `current/manifest.json`).
|
||||||
- `session status <session_id>` includes the same promoted remote output availability view as `session artifacts <session_id> --remote` when storage is configured.
|
|
||||||
- local and S3 audio input modes are mutually exclusive.
|
## Workspace and State Layout
|
||||||
- archive publish requires upstream stages through `analyze` to be `succeeded`.
|
|
||||||
- required configured artifact promotions for unselected `--artifacts` keys are skipped intentionally; selected required promotions still fail if their files are missing.
|
Session root:
|
||||||
- restore requires configured remote object storage and committed remote current state.
|
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}/`
|
||||||
|
|
||||||
|
Durable session state:
|
||||||
|
|
||||||
|
- `manifest.json`
|
||||||
|
- `inputs/**`
|
||||||
|
- `audio/**`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**`
|
||||||
|
- `reports/**`
|
||||||
|
- `logs/**`
|
||||||
|
- `config/**`
|
||||||
|
- `runs/**`
|
||||||
|
|
||||||
|
Run-local stage layout:
|
||||||
|
|
||||||
|
- `runs/{run_id}/{stage}/outputs|logs|reports|config|scratch`
|
||||||
|
|
||||||
|
Stages typically write run-local outputs first, then materialize canonical session outputs on success.
|
||||||
|
|
||||||
|
## Resume and Force Rules
|
||||||
|
|
||||||
|
- `run` and `run-stage` skip succeeded stages unless `--force` is set.
|
||||||
|
- `resume` starts at the first non-succeeded stage.
|
||||||
|
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
|
||||||
|
- `--force` does not bypass publish locks.
|
||||||
|
|
||||||
|
## Cleanup
|
||||||
|
|
||||||
|
Automatic post-publish cleanup is considered only when publish executes successfully and commits current state.
|
||||||
|
|
||||||
|
Config toggles:
|
||||||
|
|
||||||
|
- `pipeline.spool.delete_audio_after_publish=true`
|
||||||
|
- `pipeline.workspace.cleanup_after_publish=true`
|
||||||
|
|
||||||
|
Manual cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean 2026-04-04
|
||||||
|
narratio clean --all
|
||||||
|
```
|
||||||
|
|
||||||
|
Cache is preserved by default. Use `--clear-cache` to remove matching S3 audio cache entries.
|
||||||
|
|
||||||
|
## Failure and Recovery
|
||||||
|
|
||||||
|
After stage failure, Narratio keeps manifests and run-local files for inspection.
|
||||||
|
|
||||||
|
Standard recovery flow:
|
||||||
|
|
||||||
|
1. inspect status:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
2. if needed, inspect restore plan:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04 --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
3. fix root cause.
|
||||||
|
4. continue with `resume`, or rerun a stage with `--force` then `resume`.
|
||||||
|
|
||||||
|
## Operational Caveats
|
||||||
|
|
||||||
|
- local and S3 audio modes are mutually exclusive.
|
||||||
|
- publish requires prerequisite stages through `analyze` to be `succeeded`.
|
||||||
|
- restore requires configured object storage and committed current state.
|
||||||
|
- `session status` and `session artifacts --remote` both report remote published-output availability when storage is configured.
|
||||||
|
|||||||
@@ -1,17 +1,17 @@
|
|||||||
# Troubleshooting
|
# Troubleshooting
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
Canonical operator troubleshooting guide for recurring implemented Narratio failures.
|
Canonical operator troubleshooting guide for recurring Narratio failures.
|
||||||
|
|
||||||
## Config file discovery failure
|
## Config discovery failure
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `run`, `resume`, `run-stage`, `session plan`, or `session restore` fails with config/session not found.
|
- command fails because `pipeline.yml`, `campaign.yml`, or `session.yml` was not found.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- `pipeline.yml` or `session.yml` is missing from system discovery paths.
|
- missing files in discovery paths.
|
||||||
- the selected campaign ID does not exist under `pipeline.campaigns.root`.
|
- missing/incorrect campaign selection.
|
||||||
- a local working-directory config file was not passed explicitly.
|
- local file exists but was not passed explicitly.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -20,160 +20,73 @@ ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
|
|||||||
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
|
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- pass explicit `--config`, `--campaign <id>`, `--campaign-file <path>`, and `--session` as appropriate.
|
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||||
- or place files in documented discovery paths and set `pipeline.campaigns.default_campaign_id`.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
- [docs/cli.md](./cli.md)
|
|
||||||
|
|
||||||
## Templated session file rejected
|
## Templated session file rejected
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- load fails with a message that `session.yml must be concrete`.
|
- load fails because `session.yml` must be concrete.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- a template authoring file such as `session.template.yml` was passed to `--session` or uploaded as remote `session.yml`.
|
- template placeholders (`{{ ... }}`) still present in loaded session config.
|
||||||
- `session.yml` still contains `{{ ... }}` placeholders.
|
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session plan 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session ./session.yml
|
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- generate concrete YAML with `narratio session init`.
|
- generate concrete session YAML via `narratio session init`.
|
||||||
- pass the generated concrete `session.yml` to downstream commands or upload it through `session init --remote`.
|
|
||||||
|
|
||||||
Links:
|
## Strict decode or validation failure
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
|
|
||||||
## Strict YAML decode or validation failure
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- config load fails with unknown field or validation error.
|
- unknown field or invalid value error during config load.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- typo/stale field name.
|
- typo, stale field name, or invalid value.
|
||||||
- missing required fields or invalid constraints.
|
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session plan 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml
|
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- align fields/values to canonical config reference and examples.
|
- align config with [docs/config.md](./config.md) and maintained examples.
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
- [examples/](../examples/)
|
|
||||||
|
|
||||||
## `--artifacts` selection failure
|
## `--artifacts` selection failure
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `run`/`resume`/`run-stage` fails with invalid or unknown artifact selection.
|
- command fails on unknown/invalid selected artifact key.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- `--artifacts` contains blank names or unknown artifact keys.
|
- artifact key not defined in `pipeline.scriptorium.artifacts`.
|
||||||
- `pipeline.scriptorium.artifacts` missing while using `--artifacts`.
|
- empty token in `--artifacts` input.
|
||||||
|
|
||||||
Diagnostics:
|
Safe fix:
|
||||||
|
- use only configured artifact keys.
|
||||||
|
|
||||||
```bash
|
## `run-stage --artifacts` unsupported stage
|
||||||
narratio run 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --artifacts player_handout
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- use configured artifact keys only.
|
|
||||||
- ensure `pipeline.scriptorium.artifacts` is defined.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/cli.md](./cli.md)
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
|
|
||||||
## `run-stage --artifacts` on unsupported stage
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `run-stage` fails because `--artifacts` is only supported for `analyze` and `archive`.
|
- `run-stage` rejects `--artifacts` for the selected stage.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- `--artifacts` was used with a stage other than `analyze` or `archive`.
|
- `--artifacts` used with a stage other than `analyze` or `publish`.
|
||||||
|
|
||||||
Diagnostics:
|
Safe fix:
|
||||||
|
- use `--artifacts` only with `run-stage analyze ...` or `run-stage publish ...`.
|
||||||
|
|
||||||
```bash
|
## Previous-session input unavailable
|
||||||
narratio run-stage polish 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --artifacts session_recap
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- use `--artifacts` only with `run-stage analyze ...` or `run-stage archive ...`.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/cli.md](./cli.md)
|
|
||||||
|
|
||||||
## Configured artifact dependency/input validation failure
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- config validation fails for `depends_on`, `narratio.artifact.<name>` source, or artifact output path.
|
- analyze fails on required previous-session artifact input.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- `narratio.artifact.<name>` source missing matching `depends_on` key.
|
- `previous/**` cache not hydrated for this session.
|
||||||
- dependency references unknown artifact key.
|
|
||||||
- dependency self-reference or enabled dependency cycle.
|
|
||||||
- artifact output path missing/invalid/outside `artifacts/` root.
|
|
||||||
|
|
||||||
Diagnostics:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session plan 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- ensure artifact-to-artifact inputs have explicit `depends_on` entries using artifact keys.
|
|
||||||
- ensure referenced artifacts exist and define valid `output_path` values.
|
|
||||||
- keep output paths relative and under `artifacts/`.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
- [docs/internal/stage-analyze.md](./internal/stage-analyze.md)
|
|
||||||
|
|
||||||
## Required configured artifact input unavailable at analyze time
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
- analyze fails because configured input source is unavailable.
|
|
||||||
|
|
||||||
Likely Cause:
|
|
||||||
- required upstream configured artifact was not selected/executed this run.
|
|
||||||
- non-executable dependency output file is missing or invalid on disk.
|
|
||||||
|
|
||||||
Diagnostics:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session status 2026-04-04
|
|
||||||
narratio run-stage analyze 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --artifacts player_handout
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- run analyze with needed artifacts selected.
|
|
||||||
- or ensure dependency output file exists at configured path and is valid.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
|
|
||||||
## Manifest/status path failure
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
- `session status` fails because config/session state is missing, unreadable, or invalid.
|
|
||||||
|
|
||||||
Likely Cause:
|
|
||||||
- wrong session ID.
|
|
||||||
- wrong config/campaign/session file selected.
|
|
||||||
- manifest removed after cleanup.
|
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -181,137 +94,75 @@ Diagnostics:
|
|||||||
narratio session status 2026-04-04
|
narratio session status 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- use the same session ID and config files that will be used for `run`, `resume`, or `run-stage`.
|
|
||||||
|
|
||||||
Links:
|
```bash
|
||||||
- [docs/cli.md](./cli.md)
|
narratio run-stage prepare 2026-04-04 --force
|
||||||
- [docs/operations.md](./operations.md)
|
```
|
||||||
|
|
||||||
|
Or rehydrate from remote current state:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
## Session lock conflict (`.lock`)
|
## Session lock conflict (`.lock`)
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `run`, `resume`, `run-stage`, or `session restore` fails with lock conflict for session workdir.
|
- command fails with lock conflict.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- another Narratio process is running same session.
|
- another process is running for the same session.
|
||||||
- stale lock from interrupted prior run.
|
- stale lock file from interrupted command.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
||||||
cat {workspace.root}/work/{campaign}/{session_id}/.lock
|
|
||||||
ps aux | grep narratio
|
ps aux | grep narratio
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- wait for active process to finish.
|
- wait for active process; remove stale lock only if no process is active.
|
||||||
- if no process is active, remove only stale session `.lock` file.
|
|
||||||
|
|
||||||
Links:
|
## Restore current pointer/manifest missing
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
- [docs/internal/workspace.md](./internal/workspace.md)
|
|
||||||
|
|
||||||
## Restore remote current pointer or manifest missing
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `session restore` fails with remote current pointer or current manifest errors.
|
- restore fails reading remote current state.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- `current/run_id.txt` was never published.
|
- publish commit did not complete.
|
||||||
- `current/manifest.json` is missing for the session prefix.
|
- `current/run_id.txt` or `current/manifest.json` is missing.
|
||||||
- archive commit did not complete.
|
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session restore 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --dry-run
|
narratio session restore 2026-04-04 --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- verify archive stage succeeded for the target session.
|
- republish from a healthy local session state.
|
||||||
- rerun/archive from a healthy source workspace so current pointers are published.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
- [docs/internal/stage-archive.md](./internal/stage-archive.md)
|
|
||||||
|
|
||||||
## Restore manifest identity mismatch
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
- `session restore` fails because remote manifest session or campaign does not match requested values.
|
|
||||||
|
|
||||||
Likely Cause:
|
|
||||||
- wrong positional session ID or wrong session config selected.
|
|
||||||
- archive prefix points to a different campaign/session.
|
|
||||||
|
|
||||||
Diagnostics:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
narratio session restore 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- use the correct session config and positional session ID.
|
|
||||||
- verify campaign/session identity in local config before restore.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
|
|
||||||
## Restore conflict without `--force`
|
## Restore conflict without `--force`
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `session restore` fails with `restore conflict` and conflict counts.
|
- restore reports conflict and exits.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- local durable file differs from remote file for one or more planned restore paths.
|
- local durable file differs from remote restore source.
|
||||||
|
|
||||||
Diagnostics:
|
Safe fix:
|
||||||
|
- inspect with `--dry-run`.
|
||||||
|
- rerun with `--force` only when remote should overwrite local.
|
||||||
|
|
||||||
```bash
|
## Secrets or credentials failure
|
||||||
narratio session restore 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml --dry-run
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- review planned conflicts.
|
|
||||||
- rerun with `--force` only when remote state should overwrite local state.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/cli.md](./cli.md)
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
|
|
||||||
## Restore report expectations
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- operator expects restore report file but does not find one.
|
- startup fails loading secrets dir, or storage/tool auth fails at runtime.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- restore was executed in `--dry-run` mode.
|
- invalid `pipeline.secrets.env_dir`.
|
||||||
- restore failed before report persistence path (for example lock acquisition failure).
|
- missing credential env vars.
|
||||||
|
|
||||||
Diagnostics:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ls -l {workspace.root}/work/{campaign}/{session_id}/reports/restore-latest.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe Fix:
|
|
||||||
- run non-dry-run restore for durable report output.
|
|
||||||
- resolve lock or early preflight failures and retry.
|
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
|
|
||||||
## Secrets env-dir or credential-env failure
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
- startup fails loading secrets directory, or stage fails due to missing credential env vars.
|
|
||||||
|
|
||||||
Likely Cause:
|
|
||||||
- invalid `pipeline.secrets.env_dir` path/permissions.
|
|
||||||
- required credential env var unset/empty.
|
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -320,60 +171,53 @@ ls -la /path/to/secrets_dir
|
|||||||
env | grep -E 'AUDITA|OBJECT_STORAGE|AWS|SCRIPTORIUM'
|
env | grep -E 'AUDITA|OBJECT_STORAGE|AWS|SCRIPTORIUM'
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- fix secrets directory and credential env vars.
|
- fix path/permissions/env vars; keep secret values out of YAML.
|
||||||
- keep secret values out of YAML.
|
|
||||||
|
|
||||||
Links:
|
## S3 audio prepare failure
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
|
|
||||||
## S3-audio prepare failure
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- `prepare` fails in S3 mode (listing/downloading/no audio/backend error).
|
- prepare fails in S3 mode (list/download/no files/backend error).
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- wrong `session.inputs.audio_s3.prefix`.
|
- bad `session.inputs.audio_s3.prefix`.
|
||||||
- no `.flac` files at resolved prefix.
|
- no `.flac` objects at prefix.
|
||||||
- invalid/missing object-store credentials or backend config.
|
- bad storage credentials/config.
|
||||||
- mixed local+S3 audio input config.
|
- mixed local+S3 audio config.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run-stage prepare 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml
|
narratio run-stage prepare 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- configure exactly one audio source mode.
|
- configure exactly one audio mode and verify storage access.
|
||||||
- verify `.flac` files and storage access.
|
|
||||||
|
|
||||||
Links:
|
## Publish output or current-pointer failure
|
||||||
- [docs/config.md](./config.md)
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
|
|
||||||
## Archive promotion/current-pointer failure
|
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
- archive fails on required promotion source missing or pointer write failure.
|
- publish fails on required output source missing, upload error, or commit-marker write failure.
|
||||||
|
|
||||||
Likely Cause:
|
Likely cause:
|
||||||
- required promoted file absent (including analyze outputs not generated for this run).
|
- required source file not produced.
|
||||||
- storage upload failed before `current/run_id.txt` commit marker write.
|
- storage upload failed before `current/run_id.txt` write.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio session status 2026-04-04
|
narratio session status 2026-04-04
|
||||||
narratio run-stage archive 2026-04-04 --config /path/to/pipeline.yml --campaign-file /path/to/campaign.yml --session /path/to/session.yml
|
narratio run-stage publish 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe Fix:
|
Safe fix:
|
||||||
- rerun or resume upstream stages to generate required files.
|
- rerun upstream stages to regenerate required outputs.
|
||||||
- adjust promotion `source`/`dest` rules to match artifacts that must exist.
|
- adjust `pipeline.publish.outputs` source/dest rules.
|
||||||
- retry after storage issue is resolved.
|
- retry after storage issue is fixed.
|
||||||
|
|
||||||
|
## Helpful Links
|
||||||
|
|
||||||
Links:
|
|
||||||
- [docs/operations.md](./operations.md)
|
|
||||||
- [docs/config.md](./config.md)
|
- [docs/config.md](./config.md)
|
||||||
- [docs/internal/stage-archive.md](./internal/stage-archive.md)
|
- [docs/cli.md](./cli.md)
|
||||||
|
- [docs/operations.md](./operations.md)
|
||||||
|
- [docs/internal/stage-publish.md](./internal/stage-publish.md)
|
||||||
|
|||||||
@@ -4,18 +4,18 @@
|
|||||||
workspace:
|
workspace:
|
||||||
# Optional: defaults to /var/lib/narratio.
|
# Optional: defaults to /var/lib/narratio.
|
||||||
root: /var/lib/narratio/workspace
|
root: /var/lib/narratio/workspace
|
||||||
# Optional: remove run-scoped workdir after successful archive commit.
|
# Optional: remove run-scoped workdir after successful publish commit.
|
||||||
cleanup_after_archive: false
|
cleanup_after_publish: false
|
||||||
|
|
||||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||||
# secrets:
|
# secrets:
|
||||||
# env_dir: ./secrets
|
# env_dir: ./secrets
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
# Optional storage backend selector; use "s3" for archive + S3 audio workflows.
|
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
||||||
backend: s3
|
backend: s3
|
||||||
s3:
|
s3:
|
||||||
# Required when using S3 audio or S3 archive uploads.
|
# Required when using S3 audio or S3 publish uploads.
|
||||||
bucket: my-dnd-archive
|
bucket: my-dnd-archive
|
||||||
# Optional; defaults to "dnd".
|
# Optional; defaults to "dnd".
|
||||||
root_prefix: dnd
|
root_prefix: dnd
|
||||||
@@ -36,15 +36,15 @@ campaigns:
|
|||||||
spool:
|
spool:
|
||||||
# Optional; defaults to /var/spool/narratio.
|
# Optional; defaults to /var/spool/narratio.
|
||||||
root: /var/spool/narratio
|
root: /var/spool/narratio
|
||||||
# Optional cleanup of run-scoped spool audio after successful archive commit.
|
# Optional cleanup of run-scoped spool audio after successful publish commit.
|
||||||
delete_audio_after_archive: false
|
delete_audio_after_publish: false
|
||||||
|
|
||||||
archive:
|
publish:
|
||||||
# Optional booleans; defaults are true.
|
# Optional booleans; defaults are true.
|
||||||
enabled: true
|
enabled: true
|
||||||
upload_run: true
|
upload_run: true
|
||||||
# Optional promotion rules; sources use Narratio artifact source IDs.
|
# Optional publish output rules; sources use Narratio artifact source IDs.
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
workspace:
|
workspace:
|
||||||
root: /var/lib/narratio/workspace
|
root: /var/lib/narratio/workspace
|
||||||
cleanup_after_archive: true
|
cleanup_after_publish: true
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
backend: s3
|
backend: s3
|
||||||
@@ -17,12 +17,12 @@ campaigns:
|
|||||||
|
|
||||||
spool:
|
spool:
|
||||||
root: /var/spool/narratio
|
root: /var/spool/narratio
|
||||||
delete_audio_after_archive: true
|
delete_audio_after_publish: true
|
||||||
|
|
||||||
archive:
|
publish:
|
||||||
enabled: true
|
enabled: true
|
||||||
upload_run: true
|
upload_run: true
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
|||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), `run-stage: --artifacts is only supported for stages "analyze" and "archive"`) {
|
if !strings.Contains(stderr.String(), `run-stage: --artifacts is only supported for stages "analyze" and "publish"`) {
|
||||||
t.Fatalf("stderr = %q, want stage-gating error", stderr.String())
|
t.Fatalf("stderr = %q, want stage-gating error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -48,14 +48,14 @@ func TestExecuteRunStageArchivePropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
capturedStages = append(capturedStages, s.Name())
|
capturedStages = append(capturedStages, s.Name())
|
||||||
}
|
}
|
||||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"archive"}}, nil
|
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"publish"}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{
|
[]string{
|
||||||
"run-stage", "archive", "2026-05-03",
|
"run-stage", "publish", "2026-05-03",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
"--campaign-file", campaignPath,
|
"--campaign-file", campaignPath,
|
||||||
"--session", sessionPath,
|
"--session", sessionPath,
|
||||||
@@ -67,8 +67,8 @@ func TestExecuteRunStageArchivePropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if len(capturedStages) != 1 || capturedStages[0] != "archive" {
|
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||||
t.Fatalf("captured stages = %#v, want [archive]", capturedStages)
|
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||||
}
|
}
|
||||||
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||||
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||||
@@ -127,7 +127,7 @@ func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
@@ -300,7 +300,7 @@ func TestExecutePublishForceRunsArchive(t *testing.T) {
|
|||||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
return &RunSummary{
|
return &RunSummary{
|
||||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||||
Executed: []string{"archive"},
|
Executed: []string{"publish"},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -314,8 +314,8 @@ func TestExecutePublishForceRunsArchive(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if len(capturedStages) != 1 || capturedStages[0] != "archive" {
|
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||||
t.Fatalf("captured stages = %#v, want [archive]", capturedStages)
|
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||||
}
|
}
|
||||||
if !capturedForce {
|
if !capturedForce {
|
||||||
t.Fatal("captured force = false, want true")
|
t.Fatal("captured force = false, want true")
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||||
{name: "resume", args: []string{"resume", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
{name: "resume", args: []string{"resume", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
@@ -107,6 +107,22 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunStageArchiveAliasFails(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute([]string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `unknown stage "archive"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown archive stage error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
@@ -469,7 +485,7 @@ storage:
|
|||||||
backend: s3
|
backend: s3
|
||||||
s3:
|
s3:
|
||||||
bucket: test-bucket
|
bucket: test-bucket
|
||||||
archive:
|
publish:
|
||||||
enabled: true
|
enabled: true
|
||||||
upload_run: false
|
upload_run: false
|
||||||
whisperx:
|
whisperx:
|
||||||
|
|||||||
@@ -239,13 +239,13 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
if storeErr != nil {
|
if storeErr != nil {
|
||||||
fmt.Fprintf(out, "Remote archive: unavailable: %v\n", storeErr)
|
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||||
} else if store != nil {
|
} else if store != nil {
|
||||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(out, "Remote archive: missing or unavailable: %v\n", err)
|
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Fprintf(out, "Remote archive: current run %s\n", current.RunID)
|
fmt.Fprintf(out, "Remote publish: current run %s\n", current.RunID)
|
||||||
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
|
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -261,15 +261,15 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
All: staticArchiveLocks(cfg),
|
All: staticArchiveLocks(cfg),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
promotedRemoteState := map[string]string{}
|
publishedRemoteState := map[string]string{}
|
||||||
if store != nil {
|
if store != nil {
|
||||||
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||||
}
|
}
|
||||||
fmt.Fprintln(out, "Remote outputs:")
|
fmt.Fprintln(out, "Remote outputs:")
|
||||||
writeArtifactList(out, cfg, catalog, catalogLocks, promotedRemoteState)
|
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(out, "Archive locks: error: %v\n", err)
|
fmt.Fprintf(out, "Publish locks: error: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
writeLocks(out, cfg, locks)
|
writeLocks(out, cfg, locks)
|
||||||
}
|
}
|
||||||
@@ -437,11 +437,11 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("artifacts list: %w", err)
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
}
|
}
|
||||||
promotedRemoteState := map[string]string{}
|
publishedRemoteState := map[string]string{}
|
||||||
if remote && store != nil {
|
if remote && store != nil {
|
||||||
promotedRemoteState = remotePromotionAvailability(ctx, cfg, store, catalog)
|
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||||
}
|
}
|
||||||
writeArtifactList(out, cfg, catalog, locks, promotedRemoteState)
|
writeArtifactList(out, cfg, catalog, locks, publishedRemoteState)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -532,7 +532,7 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||||
@@ -542,12 +542,12 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if _, exists := remoteSet[source]; exists && !force {
|
if _, exists := remoteSet[source]; exists && !force {
|
||||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||||
}
|
}
|
||||||
remoteSet[source] = config.ArchiveLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||||
remoteLocks := lockMapValues(remoteSet)
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
if _, err := config.ValidateArchiveLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||||
@@ -589,7 +589,7 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidateArchiveLockRules([]config.ArchiveLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
remoteSet := lockSourceSet(locks.Remote)
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
@@ -601,7 +601,7 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
delete(remoteSet, source)
|
delete(remoteSet, source)
|
||||||
remoteLocks := lockMapValues(remoteSet)
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.ArchiveLockStore{Locks: remoteLocks}); err != nil {
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||||
@@ -980,7 +980,7 @@ func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog,
|
|||||||
return catalog, nil
|
return catalog, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, promotedRemoteState map[string]string) {
|
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, publishedRemoteState map[string]string) {
|
||||||
lockSet := lockSourceSet(locks.All)
|
lockSet := lockSourceSet(locks.All)
|
||||||
fmt.Fprintln(out, "Built-in:")
|
fmt.Fprintln(out, "Built-in:")
|
||||||
for _, id := range []string{
|
for _, id := range []string{
|
||||||
@@ -1000,13 +1000,13 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
|||||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||||
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
|
fmt.Fprintf(out, "- narratio.previous_session.artifact.%s required=%t\n", req.Name, req.Required)
|
||||||
}
|
}
|
||||||
fmt.Fprintln(out, "Promoted:")
|
fmt.Fprintln(out, "Published:")
|
||||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
writePromotedArtifactLine(out, rule, catalog, lockSet, promotedRemoteState)
|
writePublishedOutputLine(out, rule, catalog, lockSet, publishedRemoteState)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.ArchiveLockRule) {
|
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||||
parts := []string{source}
|
parts := []string{source}
|
||||||
if _, ok := lockSet[source]; ok {
|
if _, ok := lockSet[source]; ok {
|
||||||
parts = append(parts, "locked")
|
parts = append(parts, "locked")
|
||||||
@@ -1014,13 +1014,13 @@ func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.A
|
|||||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
}
|
}
|
||||||
|
|
||||||
func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.ArchiveLockRule, remoteState map[string]string) {
|
func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.PublishLockRule, remoteState map[string]string) {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
parts := []string{source}
|
parts := []string{source}
|
||||||
if _, ok := lockSet[source]; ok {
|
if _, ok := lockSet[source]; ok {
|
||||||
parts = append(parts, "locked")
|
parts = append(parts, "locked")
|
||||||
}
|
}
|
||||||
dest, showDest, err := helperPromotionDest(rule, catalog)
|
dest, showDest, err := helperPublishedOutputDest(rule, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
parts = append(parts, "remote=error")
|
parts = append(parts, "remote=error")
|
||||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
@@ -1029,35 +1029,35 @@ func writePromotedArtifactLine(out io.Writer, rule config.ArchivePromotionRule,
|
|||||||
if showDest {
|
if showDest {
|
||||||
parts = append(parts, "dest="+dest)
|
parts = append(parts, "dest="+dest)
|
||||||
}
|
}
|
||||||
if state := remoteState[promotionRemoteStateKey(source, dest)]; state != "" {
|
if state := remoteState[publishedOutputRemoteStateKey(source, dest)]; state != "" {
|
||||||
parts = append(parts, state)
|
parts = append(parts, state)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
}
|
}
|
||||||
|
|
||||||
func remotePromotionAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
dest, _, err := helperPromotionDest(rule, catalog)
|
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out[promotionRemoteStateKey(source, "")] = "remote=error"
|
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
key := artifacts.S3PromotedArtifactKey(sessionPrefix, dest)
|
key := artifacts.S3PublishedOutputKey(sessionPrefix, dest)
|
||||||
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||||
out[promotionRemoteStateKey(source, dest)] = "remote=promoted"
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
out[promotionRemoteStateKey(source, dest)] = "remote=error"
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=error"
|
||||||
} else {
|
} else {
|
||||||
out[promotionRemoteStateKey(source, dest)] = "remote=missing"
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func helperPromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
dest := strings.TrimSpace(rule.Dest)
|
dest := strings.TrimSpace(rule.Dest)
|
||||||
if dest == "" {
|
if dest == "" {
|
||||||
@@ -1094,20 +1094,20 @@ func normalizeHelperArchiveRelativePath(rel string) (string, error) {
|
|||||||
return cleaned, nil
|
return cleaned, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func promotionRemoteStateKey(source, dest string) string {
|
func publishedOutputRemoteStateKey(source, dest string) string {
|
||||||
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||||
if locks == nil || len(locks.All) == 0 {
|
if locks == nil || len(locks.All) == 0 {
|
||||||
fmt.Fprintln(out, "Archive locks: none")
|
fmt.Fprintln(out, "Publish locks: none")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintln(out, "Archive locks:")
|
fmt.Fprintln(out, "Publish locks:")
|
||||||
promoted := map[string]config.ArchivePromotionRule{}
|
published := map[string]config.PublishOutputRule{}
|
||||||
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Archive != nil {
|
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Publish != nil {
|
||||||
for _, rule := range cfg.Pipeline.Archive.PromoteArtifacts {
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
promoted[strings.TrimSpace(rule.Source)] = rule
|
published[strings.TrimSpace(rule.Source)] = rule
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
staticSet := lockSourceSet(locks.Static)
|
staticSet := lockSourceSet(locks.Static)
|
||||||
@@ -1116,9 +1116,9 @@ func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
|||||||
if _, ok := staticSet[lock.Source]; ok {
|
if _, ok := staticSet[lock.Source]; ok {
|
||||||
origin = "pipeline"
|
origin = "pipeline"
|
||||||
}
|
}
|
||||||
promo := "not-promoted"
|
promo := "not-published"
|
||||||
if _, ok := promoted[lock.Source]; ok {
|
if _, ok := published[lock.Source]; ok {
|
||||||
promo = "promoted"
|
promo = "published"
|
||||||
}
|
}
|
||||||
reason := strings.TrimSpace(lock.Reason)
|
reason := strings.TrimSpace(lock.Reason)
|
||||||
if reason == "" {
|
if reason == "" {
|
||||||
@@ -1128,13 +1128,13 @@ func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func lockMapValues(in map[string]config.ArchiveLockRule) []config.ArchiveLockRule {
|
func lockMapValues(in map[string]config.PublishLockRule) []config.PublishLockRule {
|
||||||
keys := make([]string, 0, len(in))
|
keys := make([]string, 0, len(in))
|
||||||
for key := range in {
|
for key := range in {
|
||||||
keys = append(keys, key)
|
keys = append(keys, key)
|
||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
out := make([]config.ArchiveLockRule, 0, len(keys))
|
out := make([]config.PublishLockRule, 0, len(keys))
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
item := in[key]
|
item := in[key]
|
||||||
item.Source = key
|
item.Source = key
|
||||||
|
|||||||
@@ -423,7 +423,7 @@ inputs:
|
|||||||
}
|
}
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
audioKey := artifacts.S3PromotedArtifactKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
|
audioKey := artifacts.S3PublishedOutputKey(artifacts.S3AudioPrefix(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "audio/"), "alice.flac")
|
||||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||||
origStoreFn := newObjectStoreFromConfigFn
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
@@ -501,9 +501,9 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(store.Locks) != 0 {
|
if len(store.Locks) != 0 {
|
||||||
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
||||||
@@ -687,13 +687,13 @@ func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
`)
|
`)
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
trimmedKey := artifacts.S3PromotedArtifactKey(
|
trimmedKey := artifacts.S3PublishedOutputKey(
|
||||||
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||||
"transcripts/final.trimmed.json",
|
"transcripts/final.trimmed.json",
|
||||||
)
|
)
|
||||||
@@ -713,7 +713,7 @@ func TestExecuteArtifactsListRemoteReportsPromotedAvailability(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "narratio.transcript.final_trimmed remote=promoted") {
|
if !strings.Contains(stdout.String(), "narratio.transcript.final_trimmed remote=published") {
|
||||||
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
|
t.Fatalf("stdout = %q, want promoted remote availability", stdout.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -722,7 +722,7 @@ func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final
|
- source: narratio.transcript.final
|
||||||
dest: transcripts/full.json
|
dest: transcripts/full.json
|
||||||
required: true
|
required: true
|
||||||
@@ -732,8 +732,8 @@ func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
|
|||||||
`)
|
`)
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||||
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PublishedOutputKey(sessionPrefix, "transcripts/full.json"), Data: []byte(`{"segments":[]}`)})
|
||||||
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3PublishedOutputKey(sessionPrefix, "transcripts/bounds.json"), Data: []byte(`{}`)})
|
||||||
var storeInitCalls int
|
var storeInitCalls int
|
||||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
@@ -759,8 +759,8 @@ func TestExecuteArtifactsListRemoteUsesPromotionDestinations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"narratio.transcript.final dest=transcripts/full.json remote=promoted",
|
"narratio.transcript.final dest=transcripts/full.json remote=published",
|
||||||
"narratio.bounds.session dest=transcripts/bounds.json remote=promoted",
|
"narratio.bounds.session dest=transcripts/bounds.json remote=published",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(out, want) {
|
if !strings.Contains(out, want) {
|
||||||
t.Fatalf("stdout = %q, want %q", out, want)
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
@@ -772,7 +772,7 @@ func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
@@ -783,8 +783,8 @@ func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
|
|||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03")
|
||||||
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
manifestKey, runIDKey := artifacts.ResolveArchiveCurrentStateKeys(sessionPrefix)
|
||||||
trimmedKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/final.trimmed.json")
|
trimmedKey := artifacts.S3PublishedOutputKey(sessionPrefix, "transcripts/final.trimmed.json")
|
||||||
fullKey := artifacts.S3PromotedArtifactKey(sessionPrefix, "transcripts/full.json")
|
fullKey := artifacts.S3PublishedOutputKey(sessionPrefix, "transcripts/full.json")
|
||||||
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
|
lockKey := artifacts.S3SessionLocksKey(sessionPrefix)
|
||||||
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
|
fake.SeedObject(storage.FakeObject{Key: runIDKey, Data: []byte("20260519T010203Z-a1b2c3d4\n")})
|
||||||
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
|
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: restoreManifestJSON(t, "2026-05-03", "sample-campaign")})
|
||||||
@@ -811,10 +811,10 @@ func TestExecuteStatusReportsRemoteArtifactCatalog(t *testing.T) {
|
|||||||
"Built-in:",
|
"Built-in:",
|
||||||
"Configured:",
|
"Configured:",
|
||||||
"Previous-session:",
|
"Previous-session:",
|
||||||
"Promoted:",
|
"Published:",
|
||||||
"narratio.transcript.final_trimmed locked",
|
"narratio.transcript.final_trimmed locked",
|
||||||
"narratio.transcript.final_trimmed locked remote=promoted",
|
"narratio.transcript.final_trimmed locked remote=published",
|
||||||
"narratio.transcript.final dest=transcripts/full.json remote=promoted",
|
"narratio.transcript.final dest=transcripts/full.json remote=published",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(out, want) {
|
if !strings.Contains(out, want) {
|
||||||
t.Fatalf("stdout = %q, want %q", out, want)
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
@@ -829,7 +829,7 @@ func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testin
|
|||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
addArchivePromotionsToPipeline(t, pipelinePath, `
|
addArchivePromotionsToPipeline(t, pipelinePath, `
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
required: true
|
required: true
|
||||||
@@ -850,13 +850,13 @@ func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testin
|
|||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
out := stdout.String()
|
out := stdout.String()
|
||||||
if !strings.Contains(out, "Remote archive: missing or unavailable:") {
|
if !strings.Contains(out, "Remote publish: missing or unavailable:") {
|
||||||
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
|
t.Fatalf("stdout = %q, want remote archive unavailable state", out)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.final_trimmed remote=error") {
|
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.final_trimmed remote=error") {
|
||||||
t.Fatalf("stdout = %q, want remote output error state", out)
|
t.Fatalf("stdout = %q, want remote output error state", out)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out, "Archive locks: error:") {
|
if !strings.Contains(out, "Publish locks: error:") {
|
||||||
t.Fatalf("stdout = %q, want archive locks error", out)
|
t.Fatalf("stdout = %q, want archive locks error", out)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -879,11 +879,11 @@ func TestExecuteArchiveLoadsRemoteLocks(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
promotedKey := artifacts.S3PromotedArtifactKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/final.trimmed.json")
|
promotedKey := artifacts.S3PublishedOutputKey(artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"), "transcripts/final.trimmed.json")
|
||||||
if _, ok := fake.Objects[promotedKey]; ok {
|
if _, ok := fake.Objects[promotedKey]; ok {
|
||||||
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
|
t.Fatalf("locked promoted key %q was uploaded", promotedKey)
|
||||||
}
|
}
|
||||||
@@ -949,8 +949,8 @@ func addStaticArchiveLockToPipelineConfig(t *testing.T, pipelinePath, source str
|
|||||||
}
|
}
|
||||||
updated := strings.Replace(
|
updated := strings.Replace(
|
||||||
string(data),
|
string(data),
|
||||||
"archive:\n enabled: true\n upload_run: false\n",
|
"publish:\n enabled: true\n upload_run: false\n",
|
||||||
"archive:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
|
"publish:\n enabled: true\n upload_run: false\n locks:\n - source: "+source+"\n reason: static review\n",
|
||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
if updated == string(data) {
|
if updated == string(data) {
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
if !strings.Contains(got, name+": run") {
|
if !strings.Contains(got, name+": run") {
|
||||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import "testing"
|
|||||||
|
|
||||||
func TestBuildFullPlanOrder(t *testing.T) {
|
func TestBuildFullPlanOrder(t *testing.T) {
|
||||||
got := BuildFullPlan()
|
got := BuildFullPlan()
|
||||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"}
|
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"}
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterArchive
|
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterPublish
|
||||||
workRequested := env.Config.Pipeline.Workspace.CleanupAfterArchive
|
workRequested := env.Config.Pipeline.Workspace.CleanupAfterPublish
|
||||||
if !spoolRequested && !workRequested {
|
if !spoolRequested && !workRequested {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -63,9 +63,9 @@ func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m
|
|||||||
}
|
}
|
||||||
|
|
||||||
if spoolRequested {
|
if spoolRequested {
|
||||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_archive"); err != nil {
|
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_publish"); err != nil {
|
||||||
sr.Metadata["cleanup_failed"] = true
|
sr.Metadata["cleanup_failed"] = true
|
||||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_archive"
|
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_publish"
|
||||||
sr.Metadata["cleanup_failed_path"] = spoolDir
|
sr.Metadata["cleanup_failed_path"] = spoolDir
|
||||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||||
return err
|
return err
|
||||||
@@ -82,9 +82,9 @@ func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_archive"); err != nil {
|
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_publish"); err != nil {
|
||||||
sr.Metadata["cleanup_failed"] = true
|
sr.Metadata["cleanup_failed"] = true
|
||||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_archive"
|
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_publish"
|
||||||
sr.Metadata["cleanup_failed_path"] = workDir
|
sr.Metadata["cleanup_failed_path"] = workDir
|
||||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||||
return err
|
return err
|
||||||
@@ -100,17 +100,17 @@ func archiveStageRecordForCleanup(m *manifest.Manifest, executed []string) *mani
|
|||||||
if m == nil {
|
if m == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
archiveRan := false
|
publishRan := false
|
||||||
for _, name := range executed {
|
for _, name := range executed {
|
||||||
if name == "archive" {
|
if name == "publish" {
|
||||||
archiveRan = true
|
publishRan = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !archiveRan {
|
if !publishRan {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
sr := m.Stages["archive"]
|
sr := m.Stages["publish"]
|
||||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -118,37 +118,37 @@ func archiveStageRecordForCleanup(m *manifest.Manifest, executed []string) *mani
|
|||||||
}
|
}
|
||||||
|
|
||||||
func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
|
func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||||
return false, "archive configuration is missing"
|
return false, "publish configuration is missing"
|
||||||
}
|
}
|
||||||
enabled := true
|
enabled := true
|
||||||
if cfg.Pipeline.Archive.Enabled != nil {
|
if cfg.Pipeline.Publish.Enabled != nil {
|
||||||
enabled = *cfg.Pipeline.Archive.Enabled
|
enabled = *cfg.Pipeline.Publish.Enabled
|
||||||
}
|
}
|
||||||
if !enabled {
|
if !enabled {
|
||||||
return false, "archive.enabled is false"
|
return false, "publish.enabled is false"
|
||||||
}
|
}
|
||||||
uploadRun := true
|
uploadRun := true
|
||||||
if cfg.Pipeline.Archive.UploadRun != nil {
|
if cfg.Pipeline.Publish.UploadRun != nil {
|
||||||
uploadRun = *cfg.Pipeline.Archive.UploadRun
|
uploadRun = *cfg.Pipeline.Publish.UploadRun
|
||||||
}
|
}
|
||||||
if !uploadRun {
|
if !uploadRun {
|
||||||
return false, "archive.upload_run is false"
|
return false, "publish.upload_run is false"
|
||||||
}
|
}
|
||||||
if sr == nil || sr.Metadata == nil {
|
if sr == nil || sr.Metadata == nil {
|
||||||
return false, "archive metadata is missing"
|
return false, "publish metadata is missing"
|
||||||
}
|
}
|
||||||
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
||||||
return false, "archive stage was skipped"
|
return false, "publish stage was skipped"
|
||||||
}
|
}
|
||||||
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||||
return false, "archive did not upload run record"
|
return false, "publish did not upload run record"
|
||||||
}
|
}
|
||||||
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
|
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
|
||||||
return false, "archive did not write current pointer"
|
return false, "publish did not write current pointer"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
|
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
|
||||||
return false, "archive current run pointer key is missing"
|
return false, "publish current run pointer key is missing"
|
||||||
}
|
}
|
||||||
return true, ""
|
return true, ""
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,11 +20,11 @@ type archiveSuccessStage struct {
|
|||||||
metadata map[string]any
|
metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
func (archiveSuccessStage) Name() string { return "archive" }
|
func (archiveSuccessStage) Name() string { return "publish" }
|
||||||
func (archiveSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
|
func (archiveSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
func (s archiveSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
func (s archiveSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
md := map[string]any{
|
md := map[string]any{
|
||||||
"stage": "archive",
|
"stage": "publish",
|
||||||
"uploaded": true,
|
"uploaded": true,
|
||||||
"current_pointer_written": true,
|
"current_pointer_written": true,
|
||||||
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
|
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
|
||||||
@@ -45,8 +45,8 @@ func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest
|
|||||||
|
|
||||||
func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -59,8 +59,8 @@ func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
|
func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -73,8 +73,8 @@ func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
|
func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -89,8 +89,8 @@ func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupBothPolicies(t *testing.T) {
|
func TestPostArchiveCleanupBothPolicies(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -104,12 +104,12 @@ func TestPostArchiveCleanupBothPolicies(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "archive", err: errors.New("archive failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "publish", err: errors.New("archive failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
if err == nil || !strings.Contains(err.Error(), "stage \"archive\" failed") {
|
if err == nil || !strings.Contains(err.Error(), "stage \"publish\" failed") {
|
||||||
t.Fatalf("executeStages() error = %v, want archive failure", err)
|
t.Fatalf("executeStages() error = %v, want publish failure", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
@@ -118,8 +118,8 @@ func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -131,8 +131,8 @@ func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -144,9 +144,9 @@ func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
cfg.Pipeline.Archive.UploadRun = boolPtr(false)
|
cfg.Pipeline.Publish.UploadRun = boolPtr(false)
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
@@ -158,8 +158,8 @@ func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
|
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
|
||||||
@@ -172,8 +172,8 @@ func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
|
func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
|
||||||
cfg, _ := cleanupFixtureConfig(t)
|
cfg, _ := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
manifestPath := manifestPathFor(cfg)
|
manifestPath := manifestPathFor(cfg)
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -194,19 +194,19 @@ func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
|
||||||
cfg, seed, runID := archiveStageCleanupFixture(t)
|
cfg, seed, runID := archiveStageCleanupFixture(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
cfg.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
cfg.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||||
}
|
}
|
||||||
|
|
||||||
archiveStageImpl, err := stage.Select("archive")
|
archiveStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Select(archive) error = %v", err)
|
t.Fatalf("Select(publish) error = %v", err)
|
||||||
}
|
}
|
||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
if err == nil || !strings.Contains(err.Error(), "required promotion source unavailable") {
|
if err == nil || !strings.Contains(err.Error(), "required output source unavailable") {
|
||||||
t.Fatalf("executeStages() error = %v, want promotion-missing failure", err)
|
t.Fatalf("executeStages() error = %v, want required output source unavailable failure", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
@@ -217,13 +217,13 @@ func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
||||||
cfg, seed, _ := archiveStageCleanupFixture(t)
|
cfg, seed, _ := archiveStageCleanupFixture(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
failKey := seed.sessionPrefix + "current/manifest.json"
|
failKey := seed.sessionPrefix + "current/manifest.json"
|
||||||
|
|
||||||
archiveStageImpl, err := stage.Select("archive")
|
archiveStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Select(archive) error = %v", err)
|
t.Fatalf("Select(publish) error = %v", err)
|
||||||
}
|
}
|
||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
||||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||||
@@ -238,13 +238,13 @@ func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
||||||
cfg, seed, _ := archiveStageCleanupFixture(t)
|
cfg, seed, _ := archiveStageCleanupFixture(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
failKey := seed.sessionPrefix + "current/run_id.txt"
|
failKey := seed.sessionPrefix + "current/run_id.txt"
|
||||||
|
|
||||||
archiveStageImpl, err := stage.Select("archive")
|
archiveStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Select(archive) error = %v", err)
|
t.Fatalf("Select(publish) error = %v", err)
|
||||||
}
|
}
|
||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
||||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||||
@@ -270,7 +270,7 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
cfg.Pipeline.Archive = &config.ArchiveConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
cfg.Pipeline.Publish = &config.PublishConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||||
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
|
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
|
||||||
|
|
||||||
runID := "20260516T010203Z-1a2b3c4d"
|
runID := "20260516T010203Z-1a2b3c4d"
|
||||||
@@ -329,10 +329,10 @@ func archiveStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, stri
|
|||||||
Bucket: "my-dnd-archive",
|
Bucket: "my-dnd-archive",
|
||||||
RootPrefix: "dnd",
|
RootPrefix: "dnd",
|
||||||
}
|
}
|
||||||
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
cfg.Pipeline.Publish = &config.PublishConfig{
|
||||||
Enabled: boolPtr(true),
|
Enabled: boolPtr(true),
|
||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
PromoteArtifacts: []config.ArchivePromotionRule{
|
Outputs: []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type effectiveLocks struct {
|
type effectiveLocks struct {
|
||||||
Static []config.ArchiveLockRule
|
Static []config.PublishLockRule
|
||||||
Remote []config.ArchiveLockRule
|
Remote []config.PublishLockRule
|
||||||
All []config.ArchiveLockRule
|
All []config.PublishLockRule
|
||||||
Key string
|
Key string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ func remoteLocksKey(cfg *config.Config) (string, error) {
|
|||||||
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.ArchiveLockStore, string, error) {
|
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, error) {
|
||||||
key, err := remoteLocksKey(cfg)
|
key, err := remoteLocksKey(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
@@ -44,7 +44,7 @@ func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.
|
|||||||
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
||||||
}
|
}
|
||||||
if !exists {
|
if !exists {
|
||||||
return &config.ArchiveLockStore{}, key, nil
|
return &config.PublishLockStore{}, key, nil
|
||||||
}
|
}
|
||||||
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
tmp, err := downloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -55,7 +55,7 @@ func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||||
}
|
}
|
||||||
lockStore, err := config.LoadArchiveLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, err
|
return nil, key, err
|
||||||
}
|
}
|
||||||
@@ -67,41 +67,41 @@ func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.O
|
|||||||
if store == nil {
|
if store == nil {
|
||||||
return &effectiveLocks{
|
return &effectiveLocks{
|
||||||
Static: staticLocks,
|
Static: staticLocks,
|
||||||
All: append([]config.ArchiveLockRule(nil), staticLocks...),
|
All: append([]config.PublishLockRule(nil), staticLocks...),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
remoteLocks := append([]config.ArchiveLockRule(nil), lockStore.Locks...)
|
remoteLocks := append([]config.PublishLockRule(nil), lockStore.Locks...)
|
||||||
return &effectiveLocks{
|
return &effectiveLocks{
|
||||||
Static: staticLocks,
|
Static: staticLocks,
|
||||||
Remote: remoteLocks,
|
Remote: remoteLocks,
|
||||||
All: config.MergeArchiveLockRules(staticLocks, remoteLocks),
|
All: config.MergePublishLockRules(staticLocks, remoteLocks),
|
||||||
Key: key,
|
Key: key,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func staticArchiveLocks(cfg *config.Config) []config.ArchiveLockRule {
|
func staticArchiveLocks(cfg *config.Config) []config.PublishLockRule {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return append([]config.ArchiveLockRule(nil), cfg.Pipeline.Archive.Locks...)
|
return append([]config.PublishLockRule(nil), cfg.Pipeline.Publish.Locks...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyEffectiveLocks(cfg *config.Config, locks []config.ArchiveLockRule) {
|
func applyEffectiveLocks(cfg *config.Config, locks []config.PublishLockRule) {
|
||||||
if cfg == nil || cfg.Pipeline == nil {
|
if cfg == nil || cfg.Pipeline == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive == nil {
|
if cfg.Pipeline.Publish == nil {
|
||||||
cfg.Pipeline.Archive = &config.ArchiveConfig{}
|
cfg.Pipeline.Publish = &config.PublishConfig{}
|
||||||
}
|
}
|
||||||
cfg.Pipeline.Archive.Locks = append([]config.ArchiveLockRule(nil), locks...)
|
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.ArchiveLockStore) error {
|
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.PublishLockStore) error {
|
||||||
data, err := config.MarshalArchiveLockStore(lockStore)
|
data, err := config.MarshalPublishLockStore(lockStore)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -124,8 +124,8 @@ func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key s
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func lockSourceSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
||||||
out := make(map[string]config.ArchiveLockRule, len(locks))
|
out := make(map[string]config.PublishLockRule, len(locks))
|
||||||
for _, lock := range locks {
|
for _, lock := range locks {
|
||||||
source := strings.TrimSpace(lock.Source)
|
source := strings.TrimSpace(lock.Source)
|
||||||
if source == "" {
|
if source == "" {
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ func TestResumeNoRemainingStages(t *testing.T) {
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
@@ -85,7 +85,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
@@ -176,7 +176,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
@@ -196,7 +196,7 @@ func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load manifest after force: %v", err)
|
t.Fatalf("load manifest after force: %v", err)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
||||||
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ func TestDecideStageActions(t *testing.T) {
|
|||||||
|
|
||||||
func TestDownstreamStageNames(t *testing.T) {
|
func TestDownstreamStageNames(t *testing.T) {
|
||||||
got := downstreamStageNames("polish")
|
got := downstreamStageNames("polish")
|
||||||
want := []string{"normalize", "trim", "analyze", "archive", "notify"}
|
want := []string{"normalize", "trim", "analyze", "publish", "notify"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
@@ -65,11 +65,11 @@ func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
|||||||
m.MarkStageSucceeded("normalize", now, nil)
|
m.MarkStageSucceeded("normalize", now, nil)
|
||||||
m.MarkStageSucceeded("trim", now, nil)
|
m.MarkStageSucceeded("trim", now, nil)
|
||||||
m.MarkStageFailed("analyze", now, "analysis failed")
|
m.MarkStageFailed("analyze", now, "analysis failed")
|
||||||
m.MarkStageSucceeded("archive", now, nil)
|
m.MarkStageSucceeded("publish", now, nil)
|
||||||
m.MarkStageSucceeded("notify", now, nil)
|
m.MarkStageSucceeded("notify", now, nil)
|
||||||
|
|
||||||
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
|
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
|
||||||
want := []string{"normalize", "trim", "archive", "notify"}
|
want := []string{"normalize", "trim", "publish", "notify"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
|
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,8 +63,8 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
|
return fmt.Errorf("run-stage: invalid --artifacts: %w", err)
|
||||||
}
|
}
|
||||||
if len(normalizedArtifacts) > 0 && stageName != "analyze" && stageName != "archive" {
|
if len(normalizedArtifacts) > 0 && stageName != "analyze" && stageName != "publish" {
|
||||||
return fmt.Errorf("run-stage: --artifacts is only supported for stages \"analyze\" and \"archive\"")
|
return fmt.Errorf("run-stage: --artifacts is only supported for stages \"analyze\" and \"publish\"")
|
||||||
}
|
}
|
||||||
|
|
||||||
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||||
@@ -164,7 +164,7 @@ func Analyze(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Publish force-runs the archive stage.
|
// Publish force-runs the publish stage.
|
||||||
func Publish(ctx context.Context, args []string, out io.Writer) error {
|
func Publish(ctx context.Context, args []string, out io.Writer) error {
|
||||||
positionalSessionID, args := pullLeadingSessionID(args)
|
positionalSessionID, args := pullLeadingSessionID(args)
|
||||||
fs := flag.NewFlagSet("publish", flag.ContinueOnError)
|
fs := flag.NewFlagSet("publish", flag.ContinueOnError)
|
||||||
@@ -209,7 +209,7 @@ func Publish(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
summary, err := runSingleStageCommand(ctx, singleStageCommand{
|
||||||
CommandName: "publish",
|
CommandName: "publish",
|
||||||
StageName: "archive",
|
StageName: "publish",
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
CampaignPath: campaignPath,
|
CampaignPath: campaignPath,
|
||||||
CampaignFilePath: campaignFilePath,
|
CampaignFilePath: campaignFilePath,
|
||||||
|
|||||||
@@ -568,16 +568,16 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !stageRequested("archive") {
|
if !stageRequested("publish") {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive == nil {
|
if cfg.Pipeline.Publish == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
if cfg.Pipeline.Publish.Enabled != nil && !*cfg.Pipeline.Publish.Enabled {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
if cfg.Pipeline.Publish.UploadRun != nil && !*cfg.Pipeline.Publish.UploadRun {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
@@ -587,23 +587,23 @@ func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
archiveRequested := false
|
publishRequested := false
|
||||||
for _, s := range stages {
|
for _, s := range stages {
|
||||||
if s != nil && s.Name() == "archive" {
|
if s != nil && s.Name() == "publish" {
|
||||||
archiveRequested = true
|
publishRequested = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !archiveRequested {
|
if !publishRequested {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive == nil {
|
if cfg.Pipeline.Publish == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
if cfg.Pipeline.Publish.Enabled != nil && !*cfg.Pipeline.Publish.Enabled {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
if cfg.Pipeline.Publish.UploadRun != nil && !*cfg.Pipeline.Publish.UploadRun {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return cfg.Pipeline.Storage.S3 != nil
|
return cfg.Pipeline.Storage.S3 != nil
|
||||||
|
|||||||
@@ -254,10 +254,10 @@ func TestExecuteStagesArchiveSkipsRequiredUnselectedConfiguredPromotion(t *testi
|
|||||||
Bucket: "my-dnd-archive",
|
Bucket: "my-dnd-archive",
|
||||||
RootPrefix: "dnd",
|
RootPrefix: "dnd",
|
||||||
}
|
}
|
||||||
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
cfg.Pipeline.Publish = &config.PublishConfig{
|
||||||
Enabled: boolPtr(true),
|
Enabled: boolPtr(true),
|
||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
PromoteArtifacts: []config.ArchivePromotionRule{
|
Outputs: []config.PublishOutputRule{
|
||||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -283,9 +283,9 @@ func TestExecuteStagesArchiveSkipsRequiredUnselectedConfiguredPromotion(t *testi
|
|||||||
t.Fatalf("Save manifest error = %v", err)
|
t.Fatalf("Save manifest error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
archiveStageImpl, err := stage.Select("archive")
|
archiveStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Select(archive) error = %v", err)
|
t.Fatalf("Select(publish) error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
summary, err := executeStages(
|
summary, err := executeStages(
|
||||||
@@ -303,7 +303,7 @@ func TestExecuteStagesArchiveSkipsRequiredUnselectedConfiguredPromotion(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(summary.Executed) != 2 || summary.Executed[0] != "analyze" || summary.Executed[1] != "archive" {
|
if len(summary.Executed) != 2 || summary.Executed[0] != "analyze" || summary.Executed[1] != "publish" {
|
||||||
t.Fatalf("executed = %#v, want analyze and archive", summary.Executed)
|
t.Fatalf("executed = %#v, want analyze and archive", summary.Executed)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -311,10 +311,10 @@ func TestExecuteStagesArchiveSkipsRequiredUnselectedConfiguredPromotion(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load manifest error = %v", err)
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
}
|
}
|
||||||
meta := loadedManifest.Stages["archive"].Metadata
|
meta := loadedManifest.Stages["publish"].Metadata
|
||||||
skipped, ok := meta["skipped_unselected_promotions"].([]any)
|
skipped, ok := meta["skipped_unselected_outputs"].([]any)
|
||||||
if !ok || len(skipped) != 1 {
|
if !ok || len(skipped) != 1 {
|
||||||
t.Fatalf("skipped_unselected_promotions = %#v, want one item", meta["skipped_unselected_promotions"])
|
t.Fatalf("skipped_unselected_outputs = %#v, want one item", meta["skipped_unselected_outputs"])
|
||||||
}
|
}
|
||||||
item, ok := skipped[0].(map[string]any)
|
item, ok := skipped[0].(map[string]any)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -342,7 +342,7 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
|||||||
t.Fatalf("Load manifest error = %v", err)
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||||
sr := m.Stages[name]
|
sr := m.Stages[name]
|
||||||
if sr == nil {
|
if sr == nil {
|
||||||
t.Fatalf("missing stage record %q", name)
|
t.Fatalf("missing stage record %q", name)
|
||||||
@@ -425,9 +425,9 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if name == "archive" {
|
if name == "publish" {
|
||||||
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
|
if sr.Metadata == nil || sr.Metadata["stage"] != "publish" {
|
||||||
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
|
t.Fatalf("archive metadata missing stage=publish: %#v", sr.Metadata)
|
||||||
}
|
}
|
||||||
if sr.Metadata["skipped"] != true {
|
if sr.Metadata["skipped"] != true {
|
||||||
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
|
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
|
||||||
@@ -522,7 +522,7 @@ func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testin
|
|||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
|
|
||||||
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "archive", "notify"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "publish", "notify"} {
|
||||||
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
||||||
@@ -553,7 +553,7 @@ func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testin
|
|||||||
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
||||||
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
||||||
}
|
}
|
||||||
for _, stageName := range []string{"normalize", "trim", "archive", "notify"} {
|
for _, stageName := range []string{"normalize", "trim", "publish", "notify"} {
|
||||||
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
||||||
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
||||||
}
|
}
|
||||||
@@ -864,7 +864,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
|
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
|
||||||
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
|
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
|
||||||
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
|
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
|
||||||
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
|
{name: "publish", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
|
||||||
{name: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("notify fail")}}},
|
{name: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("notify fail")}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -949,8 +949,8 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tc.name == "archive" {
|
if tc.name == "publish" {
|
||||||
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
cfg.Pipeline.Publish = &config.PublishConfig{
|
||||||
Enabled: boolPtr(true),
|
Enabled: boolPtr(true),
|
||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -155,13 +155,13 @@ func TestExecuteWorkflowCommandsAcceptPositionalSessionID(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "publish",
|
name: "publish",
|
||||||
args: []string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
args: []string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
wantStage: "archive",
|
wantStage: "publish",
|
||||||
wantForce: true,
|
wantForce: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "run-stage",
|
name: "run-stage",
|
||||||
args: []string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
args: []string{"run-stage", "publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
wantStage: "archive",
|
wantStage: "publish",
|
||||||
wantForce: false,
|
wantForce: false,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -246,7 +246,7 @@ func TestExecuteSessionSubcommandsAcceptPositionalSessionID(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "locks",
|
name: "locks",
|
||||||
args: []string{"session", "locks", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
args: []string{"session", "locks", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||||
want: "Archive locks:",
|
want: "Publish locks:",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -325,9 +325,9 @@ func TestExecuteSessionLocksMutationAcceptsPositionalSessionID(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
store, err := config.LoadArchiveLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(store.Locks) != 0 {
|
if len(store.Locks) != 0 {
|
||||||
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
t.Fatalf("locks after remove = %#v, want empty", store.Locks)
|
||||||
|
|||||||
@@ -58,9 +58,9 @@ func S3CurrentRunPointerKey(sessionPrefix string) string {
|
|||||||
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3RunIDFile)
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3RunIDFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
// S3PromotedArtifactKey returns the destination key for one promoted artifact.
|
// S3PublishedOutputKey returns the destination key for one published output.
|
||||||
// Format: {session_prefix}/{promotion.to}
|
// Format: {session_prefix}/{output.dest}
|
||||||
func S3PromotedArtifactKey(sessionPrefix, to string) string {
|
func S3PublishedOutputKey(sessionPrefix, to string) string {
|
||||||
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func TestS3KeyConstruction(t *testing.T) {
|
|||||||
t.Fatalf("manifest key = %q", manifestKey)
|
t.Fatalf("manifest key = %q", manifestKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
promoted := S3PromotedArtifactKey(sessionPrefix, "transcripts/final.trimmed.json")
|
promoted := S3PublishedOutputKey(sessionPrefix, "transcripts/final.trimmed.json")
|
||||||
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/final.trimmed.json" {
|
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/final.trimmed.json" {
|
||||||
t.Fatalf("promoted key = %q", promoted)
|
t.Fatalf("promoted key = %q", promoted)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ type PipelineConfig struct {
|
|||||||
Storage StorageConfig `yaml:"storage"`
|
Storage StorageConfig `yaml:"storage"`
|
||||||
Spool SpoolConfig `yaml:"spool"`
|
Spool SpoolConfig `yaml:"spool"`
|
||||||
Cache CacheConfig `yaml:"cache"`
|
Cache CacheConfig `yaml:"cache"`
|
||||||
Archive *ArchiveConfig `yaml:"archive"`
|
Publish *PublishConfig `yaml:"publish"`
|
||||||
Secrets *SecretsConfig `yaml:"secrets"`
|
Secrets *SecretsConfig `yaml:"secrets"`
|
||||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||||
Seriatim SeriatimConfig `yaml:"seriatim"`
|
Seriatim SeriatimConfig `yaml:"seriatim"`
|
||||||
@@ -65,7 +65,7 @@ type SessionConfig struct {
|
|||||||
// WorkspaceConfig configures local workspace behavior.
|
// WorkspaceConfig configures local workspace behavior.
|
||||||
type WorkspaceConfig struct {
|
type WorkspaceConfig struct {
|
||||||
Root string `yaml:"root"`
|
Root string `yaml:"root"`
|
||||||
CleanupAfterArchive bool `yaml:"cleanup_after_archive"`
|
CleanupAfterPublish bool `yaml:"cleanup_after_publish"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SecretsConfig configures optional local filesystem secret loading.
|
// SecretsConfig configures optional local filesystem secret loading.
|
||||||
@@ -93,7 +93,7 @@ type StorageS3Config struct {
|
|||||||
// SpoolConfig configures local spool storage for staged data.
|
// SpoolConfig configures local spool storage for staged data.
|
||||||
type SpoolConfig struct {
|
type SpoolConfig struct {
|
||||||
Root string `yaml:"root"`
|
Root string `yaml:"root"`
|
||||||
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CacheConfig configures durable local caches for reusable remote inputs.
|
// CacheConfig configures durable local caches for reusable remote inputs.
|
||||||
@@ -102,31 +102,31 @@ type CacheConfig struct {
|
|||||||
S3Audio *bool `yaml:"s3_audio"`
|
S3Audio *bool `yaml:"s3_audio"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveConfig configures archive behavior and artifact promotions.
|
// PublishConfig configures publish behavior and source-based output uploads.
|
||||||
type ArchiveConfig struct {
|
type PublishConfig struct {
|
||||||
Enabled *bool `yaml:"enabled"`
|
Enabled *bool `yaml:"enabled"`
|
||||||
UploadRun *bool `yaml:"upload_run"`
|
UploadRun *bool `yaml:"upload_run"`
|
||||||
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
Outputs []PublishOutputRule `yaml:"outputs"`
|
||||||
Locks []ArchiveLockRule `yaml:"locks"`
|
Locks []PublishLockRule `yaml:"locks"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchivePromotionRule configures one artifact promotion mapping.
|
// PublishOutputRule configures one source-to-destination publish mapping.
|
||||||
type ArchivePromotionRule struct {
|
type PublishOutputRule struct {
|
||||||
Source string `yaml:"source"`
|
Source string `yaml:"source"`
|
||||||
Dest string `yaml:"dest"`
|
Dest string `yaml:"dest"`
|
||||||
Required *bool `yaml:"required"`
|
Required *bool `yaml:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveLockRule prevents one source-based promotion from overwriting its
|
// PublishLockRule prevents one source from overwriting its top-level published
|
||||||
// top-level archive destination.
|
// destination.
|
||||||
type ArchiveLockRule struct {
|
type PublishLockRule struct {
|
||||||
Source string `yaml:"source"`
|
Source string `yaml:"source"`
|
||||||
Reason string `yaml:"reason"`
|
Reason string `yaml:"reason"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveLockStore is the mutable per-session remote lock store.
|
// PublishLockStore is the mutable per-session remote lock store.
|
||||||
type ArchiveLockStore struct {
|
type PublishLockStore struct {
|
||||||
Locks []ArchiveLockRule `yaml:"locks"`
|
Locks []PublishLockRule `yaml:"locks"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WhisperXConfig configures WhisperX adapter settings.
|
// WhisperXConfig configures WhisperX adapter settings.
|
||||||
|
|||||||
@@ -76,9 +76,9 @@ const (
|
|||||||
S3RunIDFile = "run_id.txt"
|
S3RunIDFile = "run_id.txt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultArchivePromoteArtifacts defines the default archive promotion rules.
|
// DefaultPublishOutputs defines the default publish output rules.
|
||||||
// Callers should copy this slice before mutating.
|
// Callers should copy this slice before mutating.
|
||||||
var DefaultArchivePromoteArtifacts = []ArchivePromotionRule{
|
var DefaultPublishOutputs = []PublishOutputRule{
|
||||||
{Source: artifactmodel.SourceTranscriptFinalTrimmed, Dest: PathTranscriptFinalTrimmed},
|
{Source: artifactmodel.SourceTranscriptFinalTrimmed, Dest: PathTranscriptFinalTrimmed},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -84,29 +84,29 @@ func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOpti
|
|||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadArchiveLockStoreBytes loads a mutable session lock store with strict
|
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
|
||||||
// field checking and source validation.
|
// field checking and source validation.
|
||||||
func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
|
func LoadPublishLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*PublishLockStore, error) {
|
||||||
var store ArchiveLockStore
|
var store PublishLockStore
|
||||||
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
||||||
return nil, fmt.Errorf("load archive lock store: %w", err)
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||||
}
|
}
|
||||||
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
|
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, "locks")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load archive lock store: %w", err)
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||||
}
|
}
|
||||||
store.Locks = locks
|
store.Locks = locks
|
||||||
return &store, nil
|
return &store, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
|
// MarshalPublishLockStore serializes a mutable lock store as strict-compatible YAML.
|
||||||
func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
|
func MarshalPublishLockStore(store *PublishLockStore) ([]byte, error) {
|
||||||
if store == nil {
|
if store == nil {
|
||||||
store = &ArchiveLockStore{}
|
store = &PublishLockStore{}
|
||||||
}
|
}
|
||||||
data, err := yaml.Marshal(store)
|
data, err := yaml.Marshal(store)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal archive lock store: %w", err)
|
return nil, fmt.Errorf("marshal publish lock store: %w", err)
|
||||||
}
|
}
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
@@ -328,7 +328,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
|||||||
applyStorageDefaults(&cfg.Storage)
|
applyStorageDefaults(&cfg.Storage)
|
||||||
applySpoolDefaults(&cfg.Spool)
|
applySpoolDefaults(&cfg.Spool)
|
||||||
applyCacheDefaults(&cfg.Cache)
|
applyCacheDefaults(&cfg.Cache)
|
||||||
applyArchiveDefaults(&cfg.Archive)
|
applyPublishDefaults(&cfg.Publish)
|
||||||
applyWhisperXDefaults(&cfg.WhisperX)
|
applyWhisperXDefaults(&cfg.WhisperX)
|
||||||
applySeriatimDefaults(&cfg.Seriatim)
|
applySeriatimDefaults(&cfg.Seriatim)
|
||||||
applyAuditaDefaults(&cfg.Audita)
|
applyAuditaDefaults(&cfg.Audita)
|
||||||
@@ -397,12 +397,12 @@ func applyCacheDefaults(cfg *CacheConfig) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
func applyPublishDefaults(cfg **PublishConfig) {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if *cfg == nil {
|
if *cfg == nil {
|
||||||
*cfg = &ArchiveConfig{}
|
*cfg = &PublishConfig{}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (*cfg).Enabled == nil {
|
if (*cfg).Enabled == nil {
|
||||||
@@ -411,12 +411,12 @@ func applyArchiveDefaults(cfg **ArchiveConfig) {
|
|||||||
if (*cfg).UploadRun == nil {
|
if (*cfg).UploadRun == nil {
|
||||||
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
||||||
}
|
}
|
||||||
if len((*cfg).PromoteArtifacts) == 0 {
|
if len((*cfg).Outputs) == 0 {
|
||||||
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
|
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
||||||
}
|
}
|
||||||
for i := range (*cfg).PromoteArtifacts {
|
for i := range (*cfg).Outputs {
|
||||||
if (*cfg).PromoteArtifacts[i].Required == nil {
|
if (*cfg).Outputs[i].Required == nil {
|
||||||
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
|
(*cfg).Outputs[i].Required = boolPtr(true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -164,33 +164,33 @@ func TestSpoolAndArchiveDefaults(t *testing.T) {
|
|||||||
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
|
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
|
||||||
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
|
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
|
if cfg.Pipeline.Spool.DeleteAudioAfterPublish {
|
||||||
t.Fatalf("spool.delete_audio_after_archive = true, want false")
|
t.Fatalf("spool.delete_audio_after_publish = true, want false")
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Workspace.CleanupAfterArchive {
|
if cfg.Pipeline.Workspace.CleanupAfterPublish {
|
||||||
t.Fatalf("workspace.cleanup_after_archive = true, want false")
|
t.Fatalf("workspace.cleanup_after_publish = true, want false")
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive == nil {
|
if cfg.Pipeline.Publish == nil {
|
||||||
t.Fatal("archive should be initialized by defaults")
|
t.Fatal("archive should be initialized by defaults")
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
|
if cfg.Pipeline.Publish.Enabled == nil || !*cfg.Pipeline.Publish.Enabled {
|
||||||
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
|
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Publish.Enabled)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
|
if cfg.Pipeline.Publish.UploadRun == nil || !*cfg.Pipeline.Publish.UploadRun {
|
||||||
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
|
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Publish.UploadRun)
|
||||||
}
|
}
|
||||||
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 1 {
|
if len(cfg.Pipeline.Publish.Outputs) != 1 {
|
||||||
t.Fatalf("archive.promote_artifacts len = %d, want 1 default", len(cfg.Pipeline.Archive.PromoteArtifacts))
|
t.Fatalf("publish.outputs len = %d, want 1 default", len(cfg.Pipeline.Publish.Outputs))
|
||||||
}
|
}
|
||||||
item := cfg.Pipeline.Archive.PromoteArtifacts[0]
|
item := cfg.Pipeline.Publish.Outputs[0]
|
||||||
if item.Required == nil || !*item.Required {
|
if item.Required == nil || !*item.Required {
|
||||||
t.Fatalf("archive.promote_artifacts[0].required = %#v, want true", item.Required)
|
t.Fatalf("publish.outputs[0].required = %#v, want true", item.Required)
|
||||||
}
|
}
|
||||||
if item.Source != "narratio.transcript.final_trimmed" {
|
if item.Source != "narratio.transcript.final_trimmed" {
|
||||||
t.Fatalf("archive.promote_artifacts[0].source = %q, want narratio.transcript.final_trimmed", item.Source)
|
t.Fatalf("publish.outputs[0].source = %q, want narratio.transcript.final_trimmed", item.Source)
|
||||||
}
|
}
|
||||||
if item.Dest != "transcripts/final.trimmed.json" {
|
if item.Dest != "transcripts/final.trimmed.json" {
|
||||||
t.Fatalf("archive.promote_artifacts[0].dest = %q, want transcripts/final.trimmed.json", item.Dest)
|
t.Fatalf("publish.outputs[0].dest = %q, want transcripts/final.trimmed.json", item.Dest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -202,8 +202,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "absolute dest path rejected",
|
name: "absolute dest path rejected",
|
||||||
ruleYML: `archive:
|
ruleYML: `publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.transcript.final_trimmed"
|
- source: "narratio.transcript.final_trimmed"
|
||||||
dest: "/transcripts/final.trimmed.json"
|
dest: "/transcripts/final.trimmed.json"
|
||||||
`,
|
`,
|
||||||
@@ -211,8 +211,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "traversal dest path rejected",
|
name: "traversal dest path rejected",
|
||||||
ruleYML: `archive:
|
ruleYML: `publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.transcript.final_trimmed"
|
- source: "narratio.transcript.final_trimmed"
|
||||||
dest: "../trimmed.json"
|
dest: "../trimmed.json"
|
||||||
`,
|
`,
|
||||||
@@ -220,8 +220,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid source rejected",
|
name: "invalid source rejected",
|
||||||
ruleYML: `archive:
|
ruleYML: `publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.unknown"
|
- source: "narratio.unknown"
|
||||||
dest: "transcripts/final.trimmed.json"
|
dest: "transcripts/final.trimmed.json"
|
||||||
`,
|
`,
|
||||||
@@ -229,19 +229,19 @@ func TestArchivePromotionValidation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "duplicate destination rejected",
|
name: "duplicate destination rejected",
|
||||||
ruleYML: `archive:
|
ruleYML: `publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.transcript.final_trimmed"
|
- source: "narratio.transcript.final_trimmed"
|
||||||
dest: "artifacts/shared.md"
|
dest: "artifacts/shared.md"
|
||||||
- source: "narratio.transcript.final"
|
- source: "narratio.transcript.final"
|
||||||
dest: "artifacts/shared.md"
|
dest: "artifacts/shared.md"
|
||||||
`,
|
`,
|
||||||
wantErr: "duplicates another archive promotion destination",
|
wantErr: "duplicates another publish output destination",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "configured source requires configured artifact key",
|
name: "configured source requires configured artifact key",
|
||||||
ruleYML: `archive:
|
ruleYML: `publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.artifact.session_recap"
|
- source: "narratio.artifact.session_recap"
|
||||||
dest: "artifacts/session_recap.md"
|
dest: "artifacts/session_recap.md"
|
||||||
`,
|
`,
|
||||||
@@ -253,8 +253,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
|||||||
artifacts:
|
artifacts:
|
||||||
session_recap:
|
session_recap:
|
||||||
enabled: false
|
enabled: false
|
||||||
archive:
|
publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: "narratio.artifact.session_recap"
|
- source: "narratio.artifact.session_recap"
|
||||||
`,
|
`,
|
||||||
wantErr: "destination cannot be derived",
|
wantErr: "destination cannot be derived",
|
||||||
@@ -288,8 +288,8 @@ func TestArchivePromotionLegacyTranscriptSourcesRejected(t *testing.T) {
|
|||||||
for _, source := range legacyTranscriptSources {
|
for _, source := range legacyTranscriptSources {
|
||||||
t.Run(source, func(t *testing.T) {
|
t.Run(source, func(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: ` + source + `
|
- source: ` + source + `
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
`
|
`
|
||||||
@@ -316,8 +316,8 @@ func TestArchivePromotionDerivesDestinationWhenOmitted(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "built in source derives canonical destination",
|
name: "built in source derives canonical destination",
|
||||||
pipelineYML: testPipelineBaseYAML + `
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.transcript.final
|
- source: narratio.transcript.final
|
||||||
`,
|
`,
|
||||||
wantDest: "transcripts/final.json",
|
wantDest: "transcripts/final.json",
|
||||||
@@ -331,8 +331,8 @@ scriptorium:
|
|||||||
enabled: true
|
enabled: true
|
||||||
prompt_id: dnd.session_recap
|
prompt_id: dnd.session_recap
|
||||||
output_path: artifacts/session_recap.md
|
output_path: artifacts/session_recap.md
|
||||||
archive:
|
publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
`,
|
`,
|
||||||
wantDest: "artifacts/session_recap.md",
|
wantDest: "artifacts/session_recap.md",
|
||||||
@@ -349,11 +349,11 @@ archive:
|
|||||||
if err := Validate(cfg); err != nil {
|
if err := Validate(cfg); err != nil {
|
||||||
t.Fatalf("Validate() error = %v", err)
|
t.Fatalf("Validate() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 1 {
|
if len(cfg.Pipeline.Publish.Outputs) != 1 {
|
||||||
t.Fatalf("archive.promote_artifacts len = %d, want 1", len(cfg.Pipeline.Archive.PromoteArtifacts))
|
t.Fatalf("publish.outputs len = %d, want 1", len(cfg.Pipeline.Publish.Outputs))
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Archive.PromoteArtifacts[0].Dest != tt.wantDest {
|
if cfg.Pipeline.Publish.Outputs[0].Dest != tt.wantDest {
|
||||||
t.Fatalf("archive.promote_artifacts[0].dest = %q, want %q", cfg.Pipeline.Archive.PromoteArtifacts[0].Dest, tt.wantDest)
|
t.Fatalf("publish.outputs[0].dest = %q, want %q", cfg.Pipeline.Publish.Outputs[0].Dest, tt.wantDest)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -368,7 +368,7 @@ func TestArchiveLockValidation(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "valid built in source",
|
name: "valid built in source",
|
||||||
pipelineYML: testPipelineBaseYAML + `
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
reason: reviewed transcript
|
reason: reviewed transcript
|
||||||
@@ -383,7 +383,7 @@ scriptorium:
|
|||||||
enabled: true
|
enabled: true
|
||||||
prompt_id: dnd.session_recap
|
prompt_id: dnd.session_recap
|
||||||
output_path: artifacts/session_recap.md
|
output_path: artifacts/session_recap.md
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
`,
|
`,
|
||||||
@@ -391,30 +391,30 @@ archive:
|
|||||||
{
|
{
|
||||||
name: "missing source rejected",
|
name: "missing source rejected",
|
||||||
pipelineYML: testPipelineBaseYAML + `
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- reason: no source
|
- reason: no source
|
||||||
`,
|
`,
|
||||||
wantErr: "pipeline.archive.locks[0].source is required",
|
wantErr: "pipeline.publish.locks[0].source is required",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid source rejected",
|
name: "invalid source rejected",
|
||||||
pipelineYML: testPipelineBaseYAML + `
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: narratio.unknown
|
- source: narratio.unknown
|
||||||
`,
|
`,
|
||||||
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
|
wantErr: "pipeline.publish.locks[0].source \"narratio.unknown\" is unsupported",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "duplicate source rejected",
|
name: "duplicate source rejected",
|
||||||
pipelineYML: testPipelineBaseYAML + `
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
- source: " narratio.transcript.final_trimmed "
|
- source: " narratio.transcript.final_trimmed "
|
||||||
`,
|
`,
|
||||||
wantErr: "duplicates another archive lock source",
|
wantErr: "duplicates another publish lock source",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -449,7 +449,7 @@ func TestArchiveLockLegacyTranscriptSourcesRejected(t *testing.T) {
|
|||||||
for _, source := range legacyTranscriptSources {
|
for _, source := range legacyTranscriptSources {
|
||||||
t.Run(source, func(t *testing.T) {
|
t.Run(source, func(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: ` + source + `
|
- source: ` + source + `
|
||||||
`
|
`
|
||||||
@@ -469,7 +469,7 @@ archive:
|
|||||||
|
|
||||||
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
locks:
|
locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
@@ -483,8 +483,8 @@ archive:
|
|||||||
|
|
||||||
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
||||||
pipelineYAML := testPipelineBaseYAML + `
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
archive:
|
publish:
|
||||||
promote_artifacts:
|
outputs:
|
||||||
- from: transcripts/final.trimmed.json
|
- from: transcripts/final.trimmed.json
|
||||||
to: transcripts/final.trimmed.json
|
to: transcripts/final.trimmed.json
|
||||||
`
|
`
|
||||||
@@ -496,18 +496,18 @@ archive:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||||
store, err := LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
store, err := LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
reason: reviewed
|
reason: reviewed
|
||||||
`), nil)
|
`), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadArchiveLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.final_trimmed" || store.Locks[0].Reason != "reviewed" {
|
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.final_trimmed" || store.Locks[0].Reason != "reviewed" {
|
||||||
t.Fatalf("locks = %#v", store.Locks)
|
t.Fatalf("locks = %#v", store.Locks)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
`), nil)
|
`), nil)
|
||||||
@@ -515,19 +515,19 @@ func TestArchiveLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
|||||||
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = LoadArchiveLockStoreBytes("locks.yml", []byte(`locks:
|
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
`), nil)
|
`), nil)
|
||||||
if err == nil || !strings.Contains(err.Error(), "duplicates another archive lock source") {
|
if err == nil || !strings.Contains(err.Error(), "duplicates another publish lock source") {
|
||||||
t.Fatalf("duplicate error = %v", err)
|
t.Fatalf("duplicate error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
|
func TestMergeArchiveLockRulesStaticWins(t *testing.T) {
|
||||||
merged := MergeArchiveLockRules(
|
merged := MergePublishLockRules(
|
||||||
[]ArchiveLockRule{{Source: "narratio.transcript.final_trimmed", Reason: "static"}},
|
[]PublishLockRule{{Source: "narratio.transcript.final_trimmed", Reason: "static"}},
|
||||||
[]ArchiveLockRule{
|
[]PublishLockRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Reason: "remote"},
|
{Source: "narratio.transcript.final_trimmed", Reason: "remote"},
|
||||||
{Source: "narratio.transcript.final", Reason: "remote full"},
|
{Source: "narratio.transcript.final", Reason: "remote full"},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
if err := validateCache(cfg.Cache); err != nil {
|
if err := validateCache(cfg.Cache); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
|
if err := validatePublish(cfg.Publish, cfg.Scriptorium); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
||||||
@@ -129,39 +129,39 @@ func validateCache(cfg CacheConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
seenDest := map[string]struct{}{}
|
seenDest := map[string]struct{}{}
|
||||||
for i, item := range cfg.PromoteArtifacts {
|
for i, item := range cfg.Outputs {
|
||||||
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
|
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||||
source := strings.TrimSpace(item.Source)
|
source := strings.TrimSpace(item.Source)
|
||||||
if source == "" {
|
if source == "" {
|
||||||
return fmt.Errorf("%s.source is required", prefix)
|
return fmt.Errorf("%s.source is required", prefix)
|
||||||
}
|
}
|
||||||
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
|
if _, err := publishSourceKnown(source, scriptorium); err != nil {
|
||||||
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
}
|
}
|
||||||
dest := strings.TrimSpace(item.Dest)
|
dest := strings.TrimSpace(item.Dest)
|
||||||
if dest == "" {
|
if dest == "" {
|
||||||
derivedDest, err := deriveArchivePromotionDest(source, scriptorium)
|
derivedDest, err := derivePublishOutputDest(source, scriptorium)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||||
}
|
}
|
||||||
dest = derivedDest
|
dest = derivedDest
|
||||||
cfg.PromoteArtifacts[i].Dest = derivedDest
|
cfg.Outputs[i].Dest = derivedDest
|
||||||
}
|
}
|
||||||
if err := validateRelativeSafePath(prefix+".dest", dest); err != nil {
|
if err := validateRelativeSafePath(prefix+".dest", dest); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
normalizedDest := filepath.ToSlash(filepath.Clean(dest))
|
normalizedDest := filepath.ToSlash(filepath.Clean(dest))
|
||||||
if _, ok := seenDest[normalizedDest]; ok {
|
if _, ok := seenDest[normalizedDest]; ok {
|
||||||
return fmt.Errorf("%s.dest %q duplicates another archive promotion destination", prefix, dest)
|
return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest)
|
||||||
}
|
}
|
||||||
seenDest[normalizedDest] = struct{}{}
|
seenDest[normalizedDest] = struct{}{}
|
||||||
}
|
}
|
||||||
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
|
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, "pipeline.publish.locks")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -169,12 +169,12 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateArchiveLockRules validates and normalizes source-based archive locks.
|
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
||||||
func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumConfig, label string) ([]ArchiveLockRule, error) {
|
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, label string) ([]PublishLockRule, error) {
|
||||||
seenLocks := map[string]struct{}{}
|
seenLocks := map[string]struct{}{}
|
||||||
out := make([]ArchiveLockRule, 0, len(locks))
|
out := make([]PublishLockRule, 0, len(locks))
|
||||||
if strings.TrimSpace(label) == "" {
|
if strings.TrimSpace(label) == "" {
|
||||||
label = "archive.locks"
|
label = "publish.locks"
|
||||||
}
|
}
|
||||||
for i, item := range locks {
|
for i, item := range locks {
|
||||||
prefix := fmt.Sprintf("%s[%d]", label, i)
|
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||||
@@ -182,14 +182,14 @@ func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumC
|
|||||||
if source == "" {
|
if source == "" {
|
||||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||||
}
|
}
|
||||||
if _, err := archiveSourceKnown(source, scriptorium); err != nil {
|
if _, err := publishSourceKnown(source, scriptorium); err != nil {
|
||||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
}
|
}
|
||||||
if _, ok := seenLocks[source]; ok {
|
if _, ok := seenLocks[source]; ok {
|
||||||
return nil, fmt.Errorf("%s.source %q duplicates another archive lock source", prefix, source)
|
return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source)
|
||||||
}
|
}
|
||||||
seenLocks[source] = struct{}{}
|
seenLocks[source] = struct{}{}
|
||||||
out = append(out, ArchiveLockRule{
|
out = append(out, PublishLockRule{
|
||||||
Source: source,
|
Source: source,
|
||||||
Reason: strings.TrimSpace(item.Reason),
|
Reason: strings.TrimSpace(item.Reason),
|
||||||
})
|
})
|
||||||
@@ -197,17 +197,17 @@ func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumC
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MergeArchiveLockRules returns the union of static and remote locks. Static
|
// MergePublishLockRules returns the union of static and remote locks. Static
|
||||||
// locks win when both sources contain the same lock.
|
// locks win when both sources contain the same lock.
|
||||||
func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []ArchiveLockRule {
|
func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule {
|
||||||
out := make([]ArchiveLockRule, 0, len(staticLocks)+len(remoteLocks))
|
out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks))
|
||||||
seen := map[string]struct{}{}
|
seen := map[string]struct{}{}
|
||||||
for _, item := range staticLocks {
|
for _, item := range staticLocks {
|
||||||
source := strings.TrimSpace(item.Source)
|
source := strings.TrimSpace(item.Source)
|
||||||
if source == "" {
|
if source == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||||
seen[source] = struct{}{}
|
seen[source] = struct{}{}
|
||||||
}
|
}
|
||||||
for _, item := range remoteLocks {
|
for _, item := range remoteLocks {
|
||||||
@@ -218,13 +218,13 @@ func MergeArchiveLockRules(staticLocks, remoteLocks []ArchiveLockRule) []Archive
|
|||||||
if _, ok := seen[source]; ok {
|
if _, ok := seen[source]; ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ArchiveLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||||
seen[source] = struct{}{}
|
seen[source] = struct{}{}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
func publishSourceKnown(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
||||||
trimmed := strings.TrimSpace(source)
|
trimmed := strings.TrimSpace(source)
|
||||||
if _, ok := artifactmodel.LookupRuntimeTranscriptArtifact(trimmed); ok {
|
if _, ok := artifactmodel.LookupRuntimeTranscriptArtifact(trimmed); ok {
|
||||||
return "", nil
|
return "", nil
|
||||||
@@ -247,7 +247,7 @@ func archiveSourceKnown(source string, scriptorium *ScriptoriumConfig) (string,
|
|||||||
return artifactKey, nil
|
return artifactKey, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func deriveArchivePromotionDest(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
func derivePublishOutputDest(source string, scriptorium *ScriptoriumConfig) (string, error) {
|
||||||
trimmed := strings.TrimSpace(source)
|
trimmed := strings.TrimSpace(source)
|
||||||
if spec, ok := artifactmodel.LookupRuntimeTranscriptArtifact(trimmed); ok {
|
if spec, ok := artifactmodel.LookupRuntimeTranscriptArtifact(trimmed); ok {
|
||||||
return spec.CanonicalRelPath, nil
|
return spec.CanonicalRelPath, nil
|
||||||
@@ -256,7 +256,7 @@ func deriveArchivePromotionDest(source string, scriptorium *ScriptoriumConfig) (
|
|||||||
case "narratio.bounds.session":
|
case "narratio.bounds.session":
|
||||||
return filepath.ToSlash(filepath.Join(PathArtifactsDirSegment, "session_bounds.json")), nil
|
return filepath.ToSlash(filepath.Join(PathArtifactsDirSegment, "session_bounds.json")), nil
|
||||||
}
|
}
|
||||||
artifactKey, err := archiveSourceKnown(trimmed, scriptorium)
|
artifactKey, err := publishSourceKnown(trimmed, scriptorium)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -629,27 +629,27 @@ func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
audioS3Enabled := session.Inputs.AudioS3 != nil
|
audioS3Enabled := session.Inputs.AudioS3 != nil
|
||||||
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
|
publishUploadEnabled := publishUploadConfiguredForS3(pipeline)
|
||||||
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
|
if (audioS3Enabled || publishUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
|
||||||
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
|
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or publish upload is enabled")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||||
if pipeline == nil || pipeline.Archive == nil {
|
if pipeline == nil || pipeline.Publish == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
|
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
enabled := true
|
enabled := true
|
||||||
if pipeline.Archive.Enabled != nil {
|
if pipeline.Publish.Enabled != nil {
|
||||||
enabled = *pipeline.Archive.Enabled
|
enabled = *pipeline.Publish.Enabled
|
||||||
}
|
}
|
||||||
upload := true
|
upload := true
|
||||||
if pipeline.Archive.UploadRun != nil {
|
if pipeline.Publish.UploadRun != nil {
|
||||||
upload = *pipeline.Archive.UploadRun
|
upload = *pipeline.Publish.UploadRun
|
||||||
}
|
}
|
||||||
return enabled && upload
|
return enabled && upload
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
InputKindManifest = "previous_manifest"
|
InputKindManifest = "previous_manifest"
|
||||||
InputKindArtifact = "previous_artifact"
|
InputKindArtifact = "previous_artifact"
|
||||||
InputSource = "previous_session_archive.current"
|
InputSource = "previous_session_publish.current"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Plan struct {
|
type Plan struct {
|
||||||
@@ -199,7 +199,7 @@ func BuildPlan(
|
|||||||
selectedRel := ""
|
selectedRel := ""
|
||||||
selectedKey := ""
|
selectedKey := ""
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
remoteKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, candidate)
|
remoteKey := artifacts.S3PublishedOutputKey(previousSessionPrefix, candidate)
|
||||||
exists, err := store.Exists(ctx, remoteKey)
|
exists, err := store.Exists(ctx, remoteKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("check previous-session artifact object %q: %w", remoteKey, err)
|
return nil, fmt.Errorf("check previous-session artifact object %q: %w", remoteKey, err)
|
||||||
@@ -289,9 +289,9 @@ func artifactRelativePathCandidates(
|
|||||||
if rel, ok := manifestArtifactRelativePathBySourceID(previousManifest, sourceID); ok {
|
if rel, ok := manifestArtifactRelativePathBySourceID(previousManifest, sourceID); ok {
|
||||||
appendCandidate(rel)
|
appendCandidate(rel)
|
||||||
base := path.Base(rel)
|
base := path.Base(rel)
|
||||||
for _, promoted := range manifestPromotedPaths(previousManifest) {
|
for _, published := range manifestPublishedPaths(previousManifest) {
|
||||||
if path.Base(promoted) == base {
|
if path.Base(published) == base {
|
||||||
appendCandidate(promoted)
|
appendCandidate(published)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -395,15 +395,15 @@ func manifestSessionRoot(previousManifest *manifest.Manifest) (string, bool) {
|
|||||||
return filepath.Dir(runsDir), true
|
return filepath.Dir(runsDir), true
|
||||||
}
|
}
|
||||||
|
|
||||||
func manifestPromotedPaths(previousManifest *manifest.Manifest) []string {
|
func manifestPublishedPaths(previousManifest *manifest.Manifest) []string {
|
||||||
if previousManifest == nil || len(previousManifest.Stages) == 0 {
|
if previousManifest == nil || len(previousManifest.Stages) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
sr := previousManifest.Stages["archive"]
|
sr := previousManifest.Stages["publish"]
|
||||||
if sr == nil || sr.Metadata == nil {
|
if sr == nil || sr.Metadata == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
raw, ok := sr.Metadata["promoted_paths"]
|
raw, ok := sr.Metadata["published_paths"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -155,10 +155,10 @@ func previousManifestWithOutput(t *testing.T, cfg *config.Config, rel string, pr
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if promoted != nil {
|
if promoted != nil {
|
||||||
if m.Stages["archive"] == nil {
|
if m.Stages["publish"] == nil {
|
||||||
m.MarkStageSucceeded("archive", time.Date(2026, 4, 26, 10, 2, 0, 0, time.UTC), nil)
|
m.MarkStageSucceeded("publish", time.Date(2026, 4, 26, 10, 2, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
m.Stages["archive"].Metadata = map[string]any{"promoted_paths": promoted}
|
m.Stages["publish"].Metadata = map[string]any{"published_paths": promoted}
|
||||||
}
|
}
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -496,13 +496,13 @@ func executeAnalyzeArtifact(
|
|||||||
if err := requireNonEmptyFile(finalOutputPath, artifactName+" output"); err != nil {
|
if err := requireNonEmptyFile(finalOutputPath, artifactName+" output"); err != nil {
|
||||||
return nil, fmt.Errorf("analyze: %w", err)
|
return nil, fmt.Errorf("analyze: %w", err)
|
||||||
}
|
}
|
||||||
promotedArtifact, err := promoteRunLocalOutput(env.ArtifactStore, finalOutputPath, canonicalOutputPath, artifacts.Ref{
|
materializedArtifact, err := materializeRunLocalOutput(env.ArtifactStore, finalOutputPath, canonicalOutputPath, artifacts.Ref{
|
||||||
Kind: artifactName,
|
Kind: artifactName,
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("analyze: promote artifact output for %q: %w", artifactName, err)
|
return nil, fmt.Errorf("analyze: materialize artifact output for %q: %w", artifactName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logPaths = append(logPaths, stdoutLogPath, stderrLogPath)
|
logPaths = append(logPaths, stdoutLogPath, stderrLogPath)
|
||||||
@@ -529,7 +529,7 @@ func executeAnalyzeArtifact(
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &analyzeArtifactExecutionResult{
|
return &analyzeArtifactExecutionResult{
|
||||||
Output: promotedArtifact,
|
Output: materializedArtifact,
|
||||||
Logs: logPaths,
|
Logs: logPaths,
|
||||||
GeneratedConfigs: generatedConfigs,
|
GeneratedConfigs: generatedConfigs,
|
||||||
Metadata: meta,
|
Metadata: meta,
|
||||||
|
|||||||
@@ -277,7 +277,7 @@ func TestAnalyzeUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
|||||||
t.Fatalf("outputs len = %d, want 1", len(result.Outputs))
|
t.Fatalf("outputs len = %d, want 1", len(result.Outputs))
|
||||||
}
|
}
|
||||||
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ var archivePrerequisiteStages = []string{
|
|||||||
"analyze",
|
"analyze",
|
||||||
}
|
}
|
||||||
|
|
||||||
func (archiveStage) Name() string { return "archive" }
|
func (archiveStage) Name() string { return "publish" }
|
||||||
|
|
||||||
func (archiveStage) Declares() IODecl {
|
func (archiveStage) Declares() IODecl {
|
||||||
return IODecl{
|
return IODecl{
|
||||||
@@ -46,13 +46,13 @@ func (archiveStage) Declares() IODecl {
|
|||||||
|
|
||||||
func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
return nil, fmt.Errorf("archive: resolved config must include pipeline and session")
|
return nil, fmt.Errorf("publish: resolved config must include pipeline and session")
|
||||||
}
|
}
|
||||||
|
|
||||||
if archiveDisabled(env) {
|
if archiveDisabled(env) {
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"stage": "archive",
|
"stage": "publish",
|
||||||
"skipped": true,
|
"skipped": true,
|
||||||
"archive_enabled": false,
|
"archive_enabled": false,
|
||||||
"audio_upload_skipped": true,
|
"audio_upload_skipped": true,
|
||||||
@@ -63,7 +63,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
if archiveRunUploadDisabled(env) {
|
if archiveRunUploadDisabled(env) {
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"stage": "archive",
|
"stage": "publish",
|
||||||
"skipped": true,
|
"skipped": true,
|
||||||
"upload_run_enabled": false,
|
"upload_run_enabled": false,
|
||||||
"audio_upload_skipped": true,
|
"audio_upload_skipped": true,
|
||||||
@@ -72,95 +72,95 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := validateArchivePrerequisites(m); err != nil {
|
if err := validatePublishPrerequisites(m); err != nil {
|
||||||
return nil, fmt.Errorf("archive: %w", err)
|
return nil, fmt.Errorf("publish: %w", err)
|
||||||
}
|
}
|
||||||
if env.ObjectStore == nil {
|
if env.ObjectStore == nil {
|
||||||
return nil, fmt.Errorf("archive: remote object store backend is required when archive run upload is enabled")
|
return nil, fmt.Errorf("publish: remote object store backend is required when publish run upload is enabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
runRoot, err := resolveArchiveRunRoot(env, m)
|
runRoot, err := resolveArchiveRunRoot(env, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve run root: %w", err)
|
return nil, fmt.Errorf("publish: resolve run root: %w", err)
|
||||||
}
|
}
|
||||||
runRootInfo, err := os.Stat(runRoot)
|
runRootInfo, err := os.Stat(runRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: run root %q: %w", runRoot, err)
|
return nil, fmt.Errorf("publish: run root %q: %w", runRoot, err)
|
||||||
}
|
}
|
||||||
if !runRootInfo.IsDir() {
|
if !runRootInfo.IsDir() {
|
||||||
return nil, fmt.Errorf("archive: run root %q is not a directory", runRoot)
|
return nil, fmt.Errorf("publish: run root %q is not a directory", runRoot)
|
||||||
}
|
}
|
||||||
|
|
||||||
runPrefix, err := artifacts.ResolveArchiveRunPrefix(env.Config, m)
|
runPrefix, err := artifacts.ResolveArchiveRunPrefix(env.Config, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve s3 run prefix: %w", err)
|
return nil, fmt.Errorf("publish: resolve s3 run prefix: %w", err)
|
||||||
}
|
}
|
||||||
sessionPrefix, err := artifacts.ResolveArchiveSessionPrefix(env.Config, m)
|
sessionPrefix, err := artifacts.ResolveArchiveSessionPrefix(env.Config, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve s3 session prefix: %w", err)
|
return nil, fmt.Errorf("publish: resolve s3 session prefix: %w", err)
|
||||||
}
|
}
|
||||||
bucket := artifacts.ResolveArchiveBucket(env.Config, m)
|
bucket := artifacts.ResolveArchiveBucket(env.Config, m)
|
||||||
if bucket == "" {
|
if bucket == "" {
|
||||||
return nil, fmt.Errorf("archive: resolve s3 bucket: bucket is required")
|
return nil, fmt.Errorf("publish: resolve s3 bucket: bucket is required")
|
||||||
}
|
}
|
||||||
runID := strings.TrimSpace(m.RunID)
|
runID := strings.TrimSpace(m.RunID)
|
||||||
if runID == "" {
|
if runID == "" {
|
||||||
return nil, fmt.Errorf("archive: run id is required")
|
return nil, fmt.Errorf("publish: run id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestSource, err := resolveArchiveRunManifestSource(runRoot)
|
manifestSource, err := resolveArchiveRunManifestSource(runRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve run manifest source: %w", err)
|
return nil, fmt.Errorf("publish: resolve run manifest source: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
runFiles, err := collectArchiveRunFiles(runRoot, manifestSource)
|
runFiles, err := collectArchiveRunFiles(runRoot, manifestSource)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: collect run files: %w", err)
|
return nil, fmt.Errorf("publish: collect run files: %w", err)
|
||||||
}
|
}
|
||||||
sessionPaths := archiveSessionPaths(env, m)
|
sessionPaths := archiveSessionPaths(env, m)
|
||||||
previousFiles, err := collectArchivePreviousFiles(sessionPaths.PreviousDir)
|
previousFiles, err := collectArchivePreviousFiles(sessionPaths.PreviousDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: collect previous files: %w", err)
|
return nil, fmt.Errorf("publish: collect previous files: %w", err)
|
||||||
}
|
}
|
||||||
runtimeCatalog, err := buildArchiveRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
runtimeCatalog, err := buildArchiveRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
return nil, fmt.Errorf("publish: build runtime artifact catalog: %w", err)
|
||||||
}
|
}
|
||||||
promotions, skippedOptional, skippedUnselected, lockedPromotions, err := resolveArchivePromotions(
|
publishOutputs, skippedOptionalOutputs, skippedUnselectedOutputs, lockedOutputs, err := resolvePublishOutputs(
|
||||||
sessionPaths,
|
sessionPaths,
|
||||||
m,
|
m,
|
||||||
runtimeCatalog,
|
runtimeCatalog,
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts,
|
env.Config.Pipeline.Publish.Outputs,
|
||||||
env.Config.Pipeline.Archive.Locks,
|
env.Config.Pipeline.Publish.Locks,
|
||||||
env.SelectedArtifactKeys,
|
env.SelectedArtifactKeys,
|
||||||
sessionPrefix,
|
sessionPrefix,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
return nil, fmt.Errorf("publish: resolve publish output rules: %w", err)
|
||||||
}
|
}
|
||||||
runUploaded := make([]string, 0, len(runFiles))
|
runUploaded := make([]string, 0, len(runFiles))
|
||||||
for _, file := range runFiles {
|
for _, file := range runFiles {
|
||||||
key := artifacts.S3RunRelativeDestinationKey(runPrefix, file.RelativePath)
|
key := artifacts.S3RunRelativeDestinationKey(runPrefix, file.RelativePath)
|
||||||
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
return nil, fmt.Errorf("archive: upload run file %q to %q: %w", file.RelativePath, key, err)
|
return nil, fmt.Errorf("publish: upload run file %q to %q: %w", file.RelativePath, key, err)
|
||||||
}
|
}
|
||||||
runUploaded = append(runUploaded, file.RelativePath)
|
runUploaded = append(runUploaded, file.RelativePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
promotedUploaded := make([]string, 0, len(promotions))
|
publishedUploaded := make([]string, 0, len(publishOutputs))
|
||||||
for _, promotion := range promotions {
|
for _, promotion := range publishOutputs {
|
||||||
key := artifacts.S3PromotedArtifactKey(sessionPrefix, promotion.Dest)
|
key := artifacts.S3PublishedOutputKey(sessionPrefix, promotion.Dest)
|
||||||
if _, err := env.ObjectStore.Upload(ctx, promotion.LocalPath, key, storage.UploadOptions{}); err != nil {
|
if _, err := env.ObjectStore.Upload(ctx, promotion.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
return nil, fmt.Errorf("archive: upload promoted output source %q to %q: %w", promotion.Source, key, err)
|
return nil, fmt.Errorf("publish: upload published output source %q to %q: %w", promotion.Source, key, err)
|
||||||
}
|
}
|
||||||
promotedUploaded = append(promotedUploaded, promotion.Dest)
|
publishedUploaded = append(publishedUploaded, promotion.Dest)
|
||||||
}
|
}
|
||||||
|
|
||||||
previousUploaded := make([]string, 0, len(previousFiles))
|
previousUploaded := make([]string, 0, len(previousFiles))
|
||||||
for _, file := range previousFiles {
|
for _, file := range previousFiles {
|
||||||
key := artifacts.S3PromotedArtifactKey(sessionPrefix, file.RelativePath)
|
key := artifacts.S3PublishedOutputKey(sessionPrefix, file.RelativePath)
|
||||||
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
return nil, fmt.Errorf("archive: upload previous file %q to %q: %w", file.RelativePath, key, err)
|
return nil, fmt.Errorf("publish: upload previous file %q to %q: %w", file.RelativePath, key, err)
|
||||||
}
|
}
|
||||||
previousUploaded = append(previousUploaded, file.RelativePath)
|
previousUploaded = append(previousUploaded, file.RelativePath)
|
||||||
}
|
}
|
||||||
@@ -171,52 +171,52 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
runPrefix,
|
runPrefix,
|
||||||
sessionPrefix,
|
sessionPrefix,
|
||||||
runUploaded,
|
runUploaded,
|
||||||
promotedUploaded,
|
publishedUploaded,
|
||||||
previousUploaded,
|
previousUploaded,
|
||||||
skippedOptional,
|
skippedOptionalOutputs,
|
||||||
skippedUnselected,
|
skippedUnselectedOutputs,
|
||||||
lockedPromotions,
|
lockedOutputs,
|
||||||
currentManifestKey,
|
currentManifestKey,
|
||||||
))
|
))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: build current manifest snapshot: %w", err)
|
return nil, fmt.Errorf("publish: build current manifest snapshot: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = os.Remove(manifestTempPath) }()
|
defer func() { _ = os.Remove(manifestTempPath) }()
|
||||||
|
|
||||||
if _, err := env.ObjectStore.Upload(ctx, manifestTempPath, currentManifestKey, storage.UploadOptions{
|
if _, err := env.ObjectStore.Upload(ctx, manifestTempPath, currentManifestKey, storage.UploadOptions{
|
||||||
ContentType: "application/json",
|
ContentType: "application/json",
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, fmt.Errorf("archive: upload current manifest to %q: %w", currentManifestKey, err)
|
return nil, fmt.Errorf("publish: upload current manifest to %q: %w", currentManifestKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
runIDTempPath, err := writeCurrentRunIDPointer(runID)
|
runIDTempPath, err := writeCurrentRunIDPointer(runID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("archive: build current run id pointer: %w", err)
|
return nil, fmt.Errorf("publish: build current run id pointer: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = os.Remove(runIDTempPath) }()
|
defer func() { _ = os.Remove(runIDTempPath) }()
|
||||||
|
|
||||||
if _, err := env.ObjectStore.Upload(ctx, runIDTempPath, currentRunPointerKey, storage.UploadOptions{
|
if _, err := env.ObjectStore.Upload(ctx, runIDTempPath, currentRunPointerKey, storage.UploadOptions{
|
||||||
ContentType: "text/plain; charset=utf-8",
|
ContentType: "text/plain; charset=utf-8",
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, fmt.Errorf("archive: upload current run pointer to %q: %w", currentRunPointerKey, err)
|
return nil, fmt.Errorf("publish: upload current run pointer to %q: %w", currentRunPointerKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"stage": "archive",
|
"stage": "publish",
|
||||||
"uploaded": true,
|
"uploaded": true,
|
||||||
"s3_bucket": bucket,
|
"s3_bucket": bucket,
|
||||||
"s3_run_prefix": runPrefix,
|
"s3_run_prefix": runPrefix,
|
||||||
"run_files_uploaded": len(runUploaded),
|
"run_files_uploaded": len(runUploaded),
|
||||||
"run_uploaded_paths": runUploaded,
|
"run_uploaded_paths": runUploaded,
|
||||||
"promoted_files_uploaded": len(promotedUploaded),
|
"published_files_uploaded": len(publishedUploaded),
|
||||||
"promoted_paths": promotedUploaded,
|
"published_paths": publishedUploaded,
|
||||||
"previous_files_uploaded": len(previousUploaded),
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
"previous_uploaded_paths": previousUploaded,
|
"previous_uploaded_paths": previousUploaded,
|
||||||
"skipped_optional_promotions": skippedOptional,
|
"skipped_optional_outputs": skippedOptionalOutputs,
|
||||||
"skipped_unselected_promotions": skippedUnselectedPromotionMetadata(skippedUnselected),
|
"skipped_unselected_outputs": skippedUnselectedOutputMetadata(skippedUnselectedOutputs),
|
||||||
"locked_promotion_count": len(lockedPromotions),
|
"locked_output_count": len(lockedOutputs),
|
||||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
"locked_outputs": lockedOutputMetadata(lockedOutputs),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": currentRunPointerKey,
|
"current_run_id_key": currentRunPointerKey,
|
||||||
"current_pointer_written": true,
|
"current_pointer_written": true,
|
||||||
@@ -225,7 +225,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type archivePromotion struct {
|
type publishOutput struct {
|
||||||
Source string
|
Source string
|
||||||
Dest string
|
Dest string
|
||||||
Required bool
|
Required bool
|
||||||
@@ -233,7 +233,7 @@ type archivePromotion struct {
|
|||||||
Provenance string
|
Provenance string
|
||||||
}
|
}
|
||||||
|
|
||||||
type archiveLockedPromotion struct {
|
type publishLockedOutput struct {
|
||||||
Source string
|
Source string
|
||||||
Dest string
|
Dest string
|
||||||
RemoteKey string
|
RemoteKey string
|
||||||
@@ -243,14 +243,14 @@ type archiveLockedPromotion struct {
|
|||||||
Provenance string
|
Provenance string
|
||||||
}
|
}
|
||||||
|
|
||||||
type archiveSkippedUnselectedPromotion struct {
|
type publishSkippedUnselectedOutput struct {
|
||||||
Source string
|
Source string
|
||||||
Dest string
|
Dest string
|
||||||
Required bool
|
Required bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func archiveDisabled(env *Env) bool {
|
func archiveDisabled(env *Env) bool {
|
||||||
cfg := env.Config.Pipeline.Archive
|
cfg := env.Config.Pipeline.Publish
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -258,14 +258,14 @@ func archiveDisabled(env *Env) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func archiveRunUploadDisabled(env *Env) bool {
|
func archiveRunUploadDisabled(env *Env) bool {
|
||||||
cfg := env.Config.Pipeline.Archive
|
cfg := env.Config.Pipeline.Publish
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return cfg.UploadRun != nil && !*cfg.UploadRun
|
return cfg.UploadRun != nil && !*cfg.UploadRun
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateArchivePrerequisites(m *manifest.Manifest) error {
|
func validatePublishPrerequisites(m *manifest.Manifest) error {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return fmt.Errorf("manifest is required")
|
return fmt.Errorf("manifest is required")
|
||||||
}
|
}
|
||||||
@@ -343,32 +343,32 @@ func archiveSessionPaths(env *Env, m *manifest.Manifest) artifacts.SessionPaths
|
|||||||
return store.SessionPathsFor(campaign, sessionID)
|
return store.SessionPathsFor(campaign, sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveArchivePromotions(
|
func resolvePublishOutputs(
|
||||||
paths artifacts.SessionPaths,
|
paths artifacts.SessionPaths,
|
||||||
m *manifest.Manifest,
|
m *manifest.Manifest,
|
||||||
catalog *artifacts.ArtifactCatalog,
|
catalog *artifacts.ArtifactCatalog,
|
||||||
rules []config.ArchivePromotionRule,
|
rules []config.PublishOutputRule,
|
||||||
locks []config.ArchiveLockRule,
|
locks []config.PublishLockRule,
|
||||||
selectedArtifactKeys []string,
|
selectedArtifactKeys []string,
|
||||||
sessionPrefix string,
|
sessionPrefix string,
|
||||||
) ([]archivePromotion, []string, []archiveSkippedUnselectedPromotion, []archiveLockedPromotion, error) {
|
) ([]publishOutput, []string, []publishSkippedUnselectedOutput, []publishLockedOutput, error) {
|
||||||
out := make([]archivePromotion, 0, len(rules))
|
out := make([]publishOutput, 0, len(rules))
|
||||||
skippedOptional := make([]string, 0)
|
skippedOptionalOutputs := make([]string, 0)
|
||||||
skippedUnselected := make([]archiveSkippedUnselectedPromotion, 0)
|
skippedUnselectedOutputs := make([]publishSkippedUnselectedOutput, 0)
|
||||||
lockedPromotions := make([]archiveLockedPromotion, 0)
|
lockedOutputs := make([]publishLockedOutput, 0)
|
||||||
lockSet := archiveLockSet(locks)
|
lockSet := archiveLockSet(locks)
|
||||||
selectedSet := archiveSelectedArtifactSet(selectedArtifactKeys)
|
selectedSet := archiveSelectedArtifactSet(selectedArtifactKeys)
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
required := rule.Required == nil || *rule.Required
|
required := rule.Required == nil || *rule.Required
|
||||||
dest, err := resolveArchivePromotionDest(rule, catalog)
|
dest, err := resolvePublishOutputDest(rule, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
return nil, nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||||
}
|
}
|
||||||
if len(selectedSet) > 0 {
|
if len(selectedSet) > 0 {
|
||||||
if key, ok := artifacts.ConfiguredArtifactName(source); ok {
|
if key, ok := artifacts.ConfiguredArtifactName(source); ok {
|
||||||
if _, selected := selectedSet[key]; !selected {
|
if _, selected := selectedSet[key]; !selected {
|
||||||
skippedUnselected = append(skippedUnselected, archiveSkippedUnselectedPromotion{
|
skippedUnselectedOutputs = append(skippedUnselectedOutputs, publishSkippedUnselectedOutput{
|
||||||
Source: source,
|
Source: source,
|
||||||
Dest: dest,
|
Dest: dest,
|
||||||
Required: required,
|
Required: required,
|
||||||
@@ -381,29 +381,29 @@ func resolveArchivePromotions(
|
|||||||
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if locked {
|
if locked {
|
||||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
lockedOutputs = append(lockedOutputs, publishLockedOutput{
|
||||||
Source: source,
|
Source: source,
|
||||||
Dest: dest,
|
Dest: dest,
|
||||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
RemoteKey: artifacts.S3PublishedOutputKey(sessionPrefix, dest),
|
||||||
Reason: strings.TrimSpace(lock.Reason),
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
Required: required,
|
Required: required,
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
||||||
skippedOptional = append(skippedOptional, dest)
|
skippedOptionalOutputs = append(skippedOptionalOutputs, dest)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
return nil, nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
return nil, nil, nil, nil, fmt.Errorf("required output source unavailable: %q", source)
|
||||||
}
|
}
|
||||||
return nil, nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
return nil, nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||||
}
|
}
|
||||||
if locked {
|
if locked {
|
||||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
lockedOutputs = append(lockedOutputs, publishLockedOutput{
|
||||||
Source: source,
|
Source: source,
|
||||||
Dest: dest,
|
Dest: dest,
|
||||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
RemoteKey: artifacts.S3PublishedOutputKey(sessionPrefix, dest),
|
||||||
Reason: strings.TrimSpace(lock.Reason),
|
Reason: strings.TrimSpace(lock.Reason),
|
||||||
Required: required,
|
Required: required,
|
||||||
LocalPath: resolved.Path,
|
LocalPath: resolved.Path,
|
||||||
@@ -411,7 +411,7 @@ func resolveArchivePromotions(
|
|||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, archivePromotion{
|
out = append(out, publishOutput{
|
||||||
Source: source,
|
Source: source,
|
||||||
Dest: dest,
|
Dest: dest,
|
||||||
Required: required,
|
Required: required,
|
||||||
@@ -419,7 +419,7 @@ func resolveArchivePromotions(
|
|||||||
Provenance: resolved.Provenance,
|
Provenance: resolved.Provenance,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out, skippedOptional, skippedUnselected, lockedPromotions, nil
|
return out, skippedOptionalOutputs, skippedUnselectedOutputs, lockedOutputs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func archiveSelectedArtifactSet(selected []string) map[string]struct{} {
|
func archiveSelectedArtifactSet(selected []string) map[string]struct{} {
|
||||||
@@ -437,8 +437,8 @@ func archiveSelectedArtifactSet(selected []string) map[string]struct{} {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
func archiveLockSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
||||||
out := make(map[string]config.ArchiveLockRule, len(locks))
|
out := make(map[string]config.PublishLockRule, len(locks))
|
||||||
for _, lock := range locks {
|
for _, lock := range locks {
|
||||||
source := strings.TrimSpace(lock.Source)
|
source := strings.TrimSpace(lock.Source)
|
||||||
if source == "" {
|
if source == "" {
|
||||||
@@ -451,7 +451,7 @@ func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLoc
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
func resolvePublishOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
||||||
dest := strings.TrimSpace(rule.Dest)
|
dest := strings.TrimSpace(rule.Dest)
|
||||||
if dest == "" {
|
if dest == "" {
|
||||||
entry, ok := catalog.Lookup(strings.TrimSpace(rule.Source))
|
entry, ok := catalog.Lookup(strings.TrimSpace(rule.Source))
|
||||||
@@ -715,8 +715,8 @@ func writeCurrentManifestSnapshot(m *manifest.Manifest, archiveMetadata map[stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
clone.MarkStageSucceeded("archive", now, nil)
|
clone.MarkStageSucceeded("publish", now, nil)
|
||||||
if sr := clone.Stages["archive"]; sr != nil {
|
if sr := clone.Stages["publish"]; sr != nil {
|
||||||
sr.Metadata = archiveMetadata
|
sr.Metadata = archiveMetadata
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -760,28 +760,28 @@ func writeCurrentRunIDPointer(runID string) (string, error) {
|
|||||||
func archiveMetadataPreview(
|
func archiveMetadataPreview(
|
||||||
bucket, runPrefix, sessionPrefix string,
|
bucket, runPrefix, sessionPrefix string,
|
||||||
runUploaded []string,
|
runUploaded []string,
|
||||||
promotedUploaded []string,
|
publishedUploaded []string,
|
||||||
previousUploaded []string,
|
previousUploaded []string,
|
||||||
skippedOptional []string,
|
skippedOptionalOutputs []string,
|
||||||
skippedUnselected []archiveSkippedUnselectedPromotion,
|
skippedUnselectedOutputs []publishSkippedUnselectedOutput,
|
||||||
lockedPromotions []archiveLockedPromotion,
|
lockedOutputs []publishLockedOutput,
|
||||||
currentManifestKey string,
|
currentManifestKey string,
|
||||||
) map[string]any {
|
) map[string]any {
|
||||||
return map[string]any{
|
return map[string]any{
|
||||||
"stage": "archive",
|
"stage": "publish",
|
||||||
"uploaded": true,
|
"uploaded": true,
|
||||||
"s3_bucket": bucket,
|
"s3_bucket": bucket,
|
||||||
"s3_run_prefix": runPrefix,
|
"s3_run_prefix": runPrefix,
|
||||||
"run_files_uploaded": len(runUploaded),
|
"run_files_uploaded": len(runUploaded),
|
||||||
"run_uploaded_paths": append([]string(nil), runUploaded...),
|
"run_uploaded_paths": append([]string(nil), runUploaded...),
|
||||||
"promoted_files_uploaded": len(promotedUploaded),
|
"published_files_uploaded": len(publishedUploaded),
|
||||||
"promoted_paths": append([]string(nil), promotedUploaded...),
|
"published_paths": append([]string(nil), publishedUploaded...),
|
||||||
"previous_files_uploaded": len(previousUploaded),
|
"previous_files_uploaded": len(previousUploaded),
|
||||||
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
||||||
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
"skipped_optional_outputs": append([]string(nil), skippedOptionalOutputs...),
|
||||||
"skipped_unselected_promotions": skippedUnselectedPromotionMetadata(skippedUnselected),
|
"skipped_unselected_outputs": skippedUnselectedOutputMetadata(skippedUnselectedOutputs),
|
||||||
"locked_promotion_count": len(lockedPromotions),
|
"locked_output_count": len(lockedOutputs),
|
||||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
"locked_outputs": lockedOutputMetadata(lockedOutputs),
|
||||||
"current_manifest_key": currentManifestKey,
|
"current_manifest_key": currentManifestKey,
|
||||||
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||||
"current_pointer_written": false,
|
"current_pointer_written": false,
|
||||||
@@ -789,7 +789,7 @@ func archiveMetadataPreview(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func skippedUnselectedPromotionMetadata(skipped []archiveSkippedUnselectedPromotion) []map[string]any {
|
func skippedUnselectedOutputMetadata(skipped []publishSkippedUnselectedOutput) []map[string]any {
|
||||||
out := make([]map[string]any, 0, len(skipped))
|
out := make([]map[string]any, 0, len(skipped))
|
||||||
for _, item := range skipped {
|
for _, item := range skipped {
|
||||||
out = append(out, map[string]any{
|
out = append(out, map[string]any{
|
||||||
@@ -801,7 +801,7 @@ func skippedUnselectedPromotionMetadata(skipped []archiveSkippedUnselectedPromot
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
|
func lockedOutputMetadata(locked []publishLockedOutput) []map[string]any {
|
||||||
out := make([]map[string]any, 0, len(locked))
|
out := make([]map[string]any, 0, len(locked))
|
||||||
for _, item := range locked {
|
for _, item := range locked {
|
||||||
out = append(out, map[string]any{
|
out = append(out, map[string]any{
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
|
|
||||||
func TestArchiveSkipsWhenDisabled(t *testing.T) {
|
func TestArchiveSkipsWhenDisabled(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.Enabled = boolPtr(false)
|
env.Config.Pipeline.Publish.Enabled = boolPtr(false)
|
||||||
|
|
||||||
result, err := archiveStage{}.Run(context.Background(), env, m)
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -35,7 +35,7 @@ func TestArchiveSkipsWhenDisabled(t *testing.T) {
|
|||||||
|
|
||||||
func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
|
func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.UploadRun = boolPtr(false)
|
env.Config.Pipeline.Publish.UploadRun = boolPtr(false)
|
||||||
|
|
||||||
result, err := archiveStage{}.Run(context.Background(), env, m)
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -128,8 +128,8 @@ func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
|
|||||||
if result.Metadata["current_pointer_written"] != true {
|
if result.Metadata["current_pointer_written"] != true {
|
||||||
t.Fatalf("metadata = %#v, want current_pointer_written=true", result.Metadata)
|
t.Fatalf("metadata = %#v, want current_pointer_written=true", result.Metadata)
|
||||||
}
|
}
|
||||||
if result.Metadata["promoted_files_uploaded"] != 2 {
|
if result.Metadata["published_files_uploaded"] != 2 {
|
||||||
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
|
t.Fatalf("metadata published_files_uploaded = %#v, want 2", result.Metadata["published_files_uploaded"])
|
||||||
}
|
}
|
||||||
if result.Metadata["previous_files_uploaded"] != 0 {
|
if result.Metadata["previous_files_uploaded"] != 0 {
|
||||||
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
||||||
@@ -179,7 +179,7 @@ func TestArchiveToleratesMissingPreviousCache(t *testing.T) {
|
|||||||
|
|
||||||
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Dest: "published/trimmed.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.final_trimmed", Dest: "published/trimmed.json", Required: boolPtr(true)},
|
||||||
{Source: "narratio.artifact.session_recap", Dest: "published/recap.md", Required: boolPtr(true)},
|
{Source: "narratio.artifact.session_recap", Dest: "published/recap.md", Required: boolPtr(true)},
|
||||||
}
|
}
|
||||||
@@ -200,7 +200,7 @@ func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
|||||||
|
|
||||||
func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(false)},
|
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(false)},
|
||||||
}
|
}
|
||||||
@@ -209,10 +209,10 @@ func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
got, _ := result.Metadata["skipped_optional_promotions"].([]string)
|
got, _ := result.Metadata["skipped_optional_outputs"].([]string)
|
||||||
want := []string{"transcripts/base.json"}
|
want := []string{"transcripts/base.json"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("skipped_optional_promotions = %#v, want %#v", got, want)
|
t.Fatalf("skipped_optional_outputs = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,15 +236,15 @@ func TestArchiveSkipsRequiredUnselectedConfiguredPromotion(t *testing.T) {
|
|||||||
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/session_recap.md"]; ok {
|
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/session_recap.md"]; ok {
|
||||||
t.Fatalf("unexpected unselected recap promotion upload")
|
t.Fatalf("unexpected unselected recap promotion upload")
|
||||||
}
|
}
|
||||||
skipped := result.Metadata["skipped_unselected_promotions"].([]map[string]any)
|
skipped := result.Metadata["skipped_unselected_outputs"].([]map[string]any)
|
||||||
if len(skipped) != 1 {
|
if len(skipped) != 1 {
|
||||||
t.Fatalf("skipped_unselected_promotions = %#v, want one item", skipped)
|
t.Fatalf("skipped_unselected_outputs = %#v, want one item", skipped)
|
||||||
}
|
}
|
||||||
if skipped[0]["source"] != "narratio.artifact.session_recap" || skipped[0]["dest"] != "artifacts/session_recap.md" || skipped[0]["required"] != true {
|
if skipped[0]["source"] != "narratio.artifact.session_recap" || skipped[0]["dest"] != "artifacts/session_recap.md" || skipped[0]["required"] != true {
|
||||||
t.Fatalf("skipped_unselected_promotions[0] = %#v, want session recap", skipped[0])
|
t.Fatalf("skipped_unselected_outputs[0] = %#v, want session recap", skipped[0])
|
||||||
}
|
}
|
||||||
if result.Metadata["locked_promotion_count"] != 0 {
|
if result.Metadata["locked_output_count"] != 0 {
|
||||||
t.Fatalf("locked_promotion_count = %#v, want 0", result.Metadata["locked_promotion_count"])
|
t.Fatalf("locked_output_count = %#v, want 0", result.Metadata["locked_output_count"])
|
||||||
}
|
}
|
||||||
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
||||||
t.Fatalf("missing current pointer")
|
t.Fatalf("missing current pointer")
|
||||||
@@ -264,15 +264,15 @@ func TestArchiveSelectedConfiguredPromotionStillFailsWhenMissing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err := archiveStage{}.Run(context.Background(), env, m)
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
if err == nil || !strings.Contains(err.Error(), `required promotion source unavailable: "narratio.artifact.session_recap"`) {
|
if err == nil || !strings.Contains(err.Error(), `required output source unavailable: "narratio.artifact.session_recap"`) {
|
||||||
t.Fatalf("Run() error = %v, want required selected promotion failure", err)
|
t.Fatalf("Run() error = %v, want required selected output failure", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveLockedSelectedPromotionSkipsAsLocked(t *testing.T) {
|
func TestArchiveLockedSelectedPromotionSkipsAsLocked(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
env.Config.Pipeline.Publish.Locks = []config.PublishLockRule{
|
||||||
{Source: "narratio.artifact.session_recap", Reason: "reviewed"},
|
{Source: "narratio.artifact.session_recap", Reason: "reviewed"},
|
||||||
}
|
}
|
||||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
@@ -284,12 +284,12 @@ func TestArchiveLockedSelectedPromotionSkipsAsLocked(t *testing.T) {
|
|||||||
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/session_recap.md"]; ok {
|
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/session_recap.md"]; ok {
|
||||||
t.Fatalf("unexpected locked recap promotion upload")
|
t.Fatalf("unexpected locked recap promotion upload")
|
||||||
}
|
}
|
||||||
if result.Metadata["locked_promotion_count"] != 1 {
|
if result.Metadata["locked_output_count"] != 1 {
|
||||||
t.Fatalf("locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
|
t.Fatalf("locked_output_count = %#v, want 1", result.Metadata["locked_output_count"])
|
||||||
}
|
}
|
||||||
skipped := result.Metadata["skipped_unselected_promotions"].([]map[string]any)
|
skipped := result.Metadata["skipped_unselected_outputs"].([]map[string]any)
|
||||||
if len(skipped) != 0 {
|
if len(skipped) != 0 {
|
||||||
t.Fatalf("skipped_unselected_promotions = %#v, want empty", skipped)
|
t.Fatalf("skipped_unselected_outputs = %#v, want empty", skipped)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,7 +301,7 @@ func TestArchiveLockedUnselectedConfiguredPromotionSkipsAsUnselected(t *testing.
|
|||||||
PromptID: "dnd.player_handout",
|
PromptID: "dnd.player_handout",
|
||||||
OutputPath: "artifacts/player_handout.md",
|
OutputPath: "artifacts/player_handout.md",
|
||||||
}
|
}
|
||||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
env.Config.Pipeline.Publish.Locks = []config.PublishLockRule{
|
||||||
{Source: "narratio.artifact.session_recap", Reason: "reviewed"},
|
{Source: "narratio.artifact.session_recap", Reason: "reviewed"},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -309,18 +309,18 @@ func TestArchiveLockedUnselectedConfiguredPromotionSkipsAsUnselected(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if result.Metadata["locked_promotion_count"] != 0 {
|
if result.Metadata["locked_output_count"] != 0 {
|
||||||
t.Fatalf("locked_promotion_count = %#v, want 0", result.Metadata["locked_promotion_count"])
|
t.Fatalf("locked_output_count = %#v, want 0", result.Metadata["locked_output_count"])
|
||||||
}
|
}
|
||||||
skipped := result.Metadata["skipped_unselected_promotions"].([]map[string]any)
|
skipped := result.Metadata["skipped_unselected_outputs"].([]map[string]any)
|
||||||
if len(skipped) != 1 || skipped[0]["source"] != "narratio.artifact.session_recap" {
|
if len(skipped) != 1 || skipped[0]["source"] != "narratio.artifact.session_recap" {
|
||||||
t.Fatalf("skipped_unselected_promotions = %#v, want unselected recap", skipped)
|
t.Fatalf("skipped_unselected_outputs = %#v, want unselected recap", skipped)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
env.Config.Pipeline.Publish.Locks = []config.PublishLockRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Reason: "human reviewed"},
|
{Source: "narratio.transcript.final_trimmed", Reason: "human reviewed"},
|
||||||
}
|
}
|
||||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
@@ -348,15 +348,15 @@ func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
|||||||
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
|
t.Fatalf("last upload = %#v, want current run pointer %q", fake.Uploads, currentRunIDKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.Metadata["promoted_files_uploaded"] != 1 {
|
if result.Metadata["published_files_uploaded"] != 1 {
|
||||||
t.Fatalf("metadata promoted_files_uploaded = %#v, want 1", result.Metadata["promoted_files_uploaded"])
|
t.Fatalf("metadata published_files_uploaded = %#v, want 1", result.Metadata["published_files_uploaded"])
|
||||||
}
|
}
|
||||||
if result.Metadata["locked_promotion_count"] != 1 {
|
if result.Metadata["locked_output_count"] != 1 {
|
||||||
t.Fatalf("metadata locked_promotion_count = %#v, want 1", result.Metadata["locked_promotion_count"])
|
t.Fatalf("metadata locked_output_count = %#v, want 1", result.Metadata["locked_output_count"])
|
||||||
}
|
}
|
||||||
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
locked := result.Metadata["locked_outputs"].([]map[string]any)
|
||||||
if len(locked) != 1 {
|
if len(locked) != 1 {
|
||||||
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
t.Fatalf("locked_outputs = %#v, want one item", locked)
|
||||||
}
|
}
|
||||||
if locked[0]["source"] != "narratio.transcript.final_trimmed" ||
|
if locked[0]["source"] != "narratio.transcript.final_trimmed" ||
|
||||||
locked[0]["dest"] != "transcripts/final.trimmed.json" ||
|
locked[0]["dest"] != "transcripts/final.trimmed.json" ||
|
||||||
@@ -374,23 +374,23 @@ func TestArchiveSkipsLockedRequiredPromotionAndCommits(t *testing.T) {
|
|||||||
t.Fatalf("unmarshal current manifest: %v", err)
|
t.Fatalf("unmarshal current manifest: %v", err)
|
||||||
}
|
}
|
||||||
stages := current["stages"].(map[string]any)
|
stages := current["stages"].(map[string]any)
|
||||||
archive := stages["archive"].(map[string]any)
|
archive := stages["publish"].(map[string]any)
|
||||||
meta := archive["metadata"].(map[string]any)
|
meta := archive["metadata"].(map[string]any)
|
||||||
if meta["locked_promotion_count"] != float64(1) {
|
if meta["locked_output_count"] != float64(1) {
|
||||||
t.Fatalf("current manifest locked_promotion_count = %#v, want 1", meta["locked_promotion_count"])
|
t.Fatalf("current manifest locked_output_count = %#v, want 1", meta["locked_output_count"])
|
||||||
}
|
}
|
||||||
items := meta["locked_promotions"].([]any)
|
items := meta["locked_outputs"].([]any)
|
||||||
if len(items) != 1 {
|
if len(items) != 1 {
|
||||||
t.Fatalf("current manifest locked_promotions = %#v, want one item", items)
|
t.Fatalf("current manifest locked_outputs = %#v, want one item", items)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||||
}
|
}
|
||||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
env.Config.Pipeline.Publish.Locks = []config.PublishLockRule{
|
||||||
{Source: "narratio.transcript.base", Reason: "manual merge is locked"},
|
{Source: "narratio.transcript.base", Reason: "manual merge is locked"},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -407,9 +407,9 @@ func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
|||||||
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; !ok {
|
||||||
t.Fatalf("current run pointer should be written for locked missing promotion")
|
t.Fatalf("current run pointer should be written for locked missing promotion")
|
||||||
}
|
}
|
||||||
locked := result.Metadata["locked_promotions"].([]map[string]any)
|
locked := result.Metadata["locked_outputs"].([]map[string]any)
|
||||||
if len(locked) != 1 {
|
if len(locked) != 1 {
|
||||||
t.Fatalf("locked_promotions = %#v, want one item", locked)
|
t.Fatalf("locked_outputs = %#v, want one item", locked)
|
||||||
}
|
}
|
||||||
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
|
if locked[0]["local_path"] != "" || locked[0]["provenance"] != "" {
|
||||||
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
|
t.Fatalf("locked missing promotion metadata = %#v, want empty local path/provenance", locked[0])
|
||||||
@@ -418,7 +418,7 @@ func TestArchiveLockedRequiredMissingPromotionSucceeds(t *testing.T) {
|
|||||||
|
|
||||||
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.Locks = []config.ArchiveLockRule{
|
env.Config.Pipeline.Publish.Locks = []config.PublishLockRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Reason: "already published"},
|
{Source: "narratio.transcript.final_trimmed", Reason: "already published"},
|
||||||
}
|
}
|
||||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
@@ -442,13 +442,13 @@ func TestArchiveLockDoesNotOverwriteExistingPromotion(t *testing.T) {
|
|||||||
|
|
||||||
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
||||||
env, m, _ := archiveFixture(t)
|
env, m, _ := archiveFixture(t)
|
||||||
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := archiveStage{}.Run(context.Background(), env, m)
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
if err == nil || !strings.Contains(err.Error(), "required promotion source unavailable") {
|
if err == nil || !strings.Contains(err.Error(), "required output source unavailable") {
|
||||||
t.Fatalf("Run() error = %v, want required promotion source unavailable failure", err)
|
t.Fatalf("Run() error = %v, want required output source unavailable failure", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -485,8 +485,8 @@ func TestArchiveDoesNotWriteCurrentPointerWhenPromotionUploadFails(t *testing.T)
|
|||||||
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: failingKey}
|
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: failingKey}
|
||||||
|
|
||||||
_, err := archiveStage{}.Run(context.Background(), env, m)
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
if err == nil || !strings.Contains(err.Error(), "promoted output") {
|
if err == nil || !strings.Contains(err.Error(), "upload published output source") {
|
||||||
t.Fatalf("Run() error = %v, want promotion upload failure", err)
|
t.Fatalf("Run() error = %v, want published output upload failure", err)
|
||||||
}
|
}
|
||||||
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
|
||||||
t.Fatalf("unexpected current pointer write on promotion failure")
|
t.Fatalf("unexpected current pointer write on promotion failure")
|
||||||
@@ -563,10 +563,10 @@ func archiveFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
|||||||
RootPrefix: "dnd",
|
RootPrefix: "dnd",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Archive: &config.ArchiveConfig{
|
Publish: &config.PublishConfig{
|
||||||
Enabled: boolPtr(true),
|
Enabled: boolPtr(true),
|
||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
PromoteArtifacts: []config.ArchivePromotionRule{
|
Outputs: []config.PublishOutputRule{
|
||||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -150,7 +150,7 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
promotedMerged, err := promoteRunLocalOutput(env.ArtifactStore, finalMergedPath, canonicalMergedPath, artifacts.Ref{
|
materializedMerged, err := materializeRunLocalOutput(env.ArtifactStore, finalMergedPath, canonicalMergedPath, artifacts.Ref{
|
||||||
Kind: "transcript_base",
|
Kind: "transcript_base",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -158,9 +158,9 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("merge: promote merged transcript: %w", err)
|
return nil, fmt.Errorf("merge: promote merged transcript: %w", err)
|
||||||
}
|
}
|
||||||
outputs := []artifacts.Ref{promotedMerged}
|
outputs := []artifacts.Ref{materializedMerged}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
materializedReport, err := materializeRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "seriatim_report",
|
Kind: "seriatim_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -168,7 +168,7 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("merge: promote report: %w", err)
|
return nil, fmt.Errorf("merge: promote report: %w", err)
|
||||||
}
|
}
|
||||||
outputs = append(outputs, promotedReport)
|
outputs = append(outputs, materializedReport)
|
||||||
}
|
}
|
||||||
|
|
||||||
coalesceGap := any(nil)
|
coalesceGap := any(nil)
|
||||||
|
|||||||
@@ -315,10 +315,10 @@ func TestMergeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
|||||||
t.Fatalf("run output path = %q, want run-local path", req.OutputMergedTranscriptPath)
|
t.Fatalf("run output path = %q, want run-local path", req.OutputMergedTranscriptPath)
|
||||||
}
|
}
|
||||||
if len(result.Outputs) == 0 {
|
if len(result.Outputs) == 0 {
|
||||||
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
t.Fatalf("outputs = %#v, want materialized outputs", result.Outputs)
|
||||||
}
|
}
|
||||||
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
promotedNormalized, err := promoteRunLocalOutput(env.ArtifactStore, finalNormalizedPath, canonicalNormalizedPath, artifacts.Ref{
|
materializedNormalized, err := materializeRunLocalOutput(env.ArtifactStore, finalNormalizedPath, canonicalNormalizedPath, artifacts.Ref{
|
||||||
Kind: "transcript_final",
|
Kind: "transcript_final",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -141,9 +141,9 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("normalize: promote normalized transcript: %w", err)
|
return nil, fmt.Errorf("normalize: promote normalized transcript: %w", err)
|
||||||
}
|
}
|
||||||
outputs := []artifacts.Ref{promotedNormalized}
|
outputs := []artifacts.Ref{materializedNormalized}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
materializedReport, err := materializeRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "seriatim_normalize_report",
|
Kind: "seriatim_normalize_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -151,7 +151,7 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("normalize: promote report: %w", err)
|
return nil, fmt.Errorf("normalize: promote report: %w", err)
|
||||||
}
|
}
|
||||||
outputs = append(outputs, promotedReport)
|
outputs = append(outputs, materializedReport)
|
||||||
}
|
}
|
||||||
reportCanonicalPath := ""
|
reportCanonicalPath := ""
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
|
|||||||
@@ -231,10 +231,10 @@ func TestNormalizeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
|||||||
t.Fatalf("run output path = %q, want run-local path", req.OutputNormalizedPath)
|
t.Fatalf("run output path = %q, want run-local path", req.OutputNormalizedPath)
|
||||||
}
|
}
|
||||||
if len(result.Outputs) == 0 {
|
if len(result.Outputs) == 0 {
|
||||||
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
t.Fatalf("outputs = %#v, want materialized outputs", result.Outputs)
|
||||||
}
|
}
|
||||||
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
RootPrefix: "dnd",
|
RootPrefix: "dnd",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Archive: &config.ArchiveConfig{
|
Publish: &config.PublishConfig{
|
||||||
Enabled: boolPtr(true),
|
Enabled: boolPtr(true),
|
||||||
UploadRun: boolPtr(true),
|
UploadRun: boolPtr(true),
|
||||||
},
|
},
|
||||||
@@ -195,9 +195,9 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if s.Name() == "archive" {
|
if s.Name() == "publish" {
|
||||||
if result.Metadata["stage"] != "archive" {
|
if result.Metadata["stage"] != "publish" {
|
||||||
t.Fatalf("archive metadata = %#v, want stage=archive", result.Metadata)
|
t.Fatalf("archive metadata = %#v, want stage=publish", result.Metadata)
|
||||||
}
|
}
|
||||||
if result.Metadata["uploaded"] != true {
|
if result.Metadata["uploaded"] != true {
|
||||||
t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
|
t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
|
||||||
|
|||||||
@@ -149,7 +149,7 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
promotedProcessed, err := promoteRunLocalOutput(env.ArtifactStore, finalProcessedPath, canonicalProcessedPath, artifacts.Ref{
|
materializedProcessed, err := materializeRunLocalOutput(env.ArtifactStore, finalProcessedPath, canonicalProcessedPath, artifacts.Ref{
|
||||||
Kind: "transcript_polished",
|
Kind: "transcript_polished",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -157,9 +157,9 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("polish: promote processed transcript: %w", err)
|
return nil, fmt.Errorf("polish: promote processed transcript: %w", err)
|
||||||
}
|
}
|
||||||
outputs := []artifacts.Ref{promotedProcessed}
|
outputs := []artifacts.Ref{materializedProcessed}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
materializedReport, err := materializeRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "audita_report",
|
Kind: "audita_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -167,7 +167,7 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("polish: promote report: %w", err)
|
return nil, fmt.Errorf("polish: promote report: %w", err)
|
||||||
}
|
}
|
||||||
outputs = append(outputs, promotedReport)
|
outputs = append(outputs, materializedReport)
|
||||||
}
|
}
|
||||||
|
|
||||||
var validationConcurrency any
|
var validationConcurrency any
|
||||||
|
|||||||
@@ -261,10 +261,10 @@ func TestPolishStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
|||||||
t.Fatalf("run output path = %q, want run-local path", req.OutputProcessedPath)
|
t.Fatalf("run output path = %q, want run-local path", req.OutputProcessedPath)
|
||||||
}
|
}
|
||||||
if len(result.Outputs) == 0 {
|
if len(result.Outputs) == 0 {
|
||||||
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
t.Fatalf("outputs = %#v, want materialized outputs", result.Outputs)
|
||||||
}
|
}
|
||||||
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -300,7 +300,7 @@ func seedPreviousCurrentState(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
artifactKey := artifacts.S3PromotedArtifactKey(previousSessionPrefix, "artifacts/session_recap.md")
|
artifactKey := artifacts.S3PublishedOutputKey(previousSessionPrefix, "artifacts/session_recap.md")
|
||||||
if options.includeArtifactObject {
|
if options.includeArtifactObject {
|
||||||
body := options.artifactBody
|
body := options.artifactBody
|
||||||
if body == "" {
|
if body == "" {
|
||||||
@@ -365,9 +365,9 @@ func buildPreviousManifestForSeed(
|
|||||||
LocalPath: artifactPath,
|
LocalPath: artifactPath,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
m.MarkStageSucceeded("archive", now, nil)
|
m.MarkStageSucceeded("publish", now, nil)
|
||||||
m.Stages["archive"].Metadata = map[string]any{
|
m.Stages["publish"].Metadata = map[string]any{
|
||||||
"promoted_paths": []string{"artifacts/session_recap.md"},
|
"published_paths": []string{"artifacts/session_recap.md"},
|
||||||
}
|
}
|
||||||
data, err := json.MarshalIndent(m, "", " ")
|
data, err := json.MarshalIndent(m, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ func runLocalPathForCanonical(layout runStageLayout, sessionPaths artifacts.Sess
|
|||||||
return localPath, nil
|
return localPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func promoteRunLocalOutput(
|
func materializeRunLocalOutput(
|
||||||
store artifacts.Store,
|
store artifacts.Store,
|
||||||
srcPath, canonicalPath string,
|
srcPath, canonicalPath string,
|
||||||
ref artifacts.Ref,
|
ref artifacts.Ref,
|
||||||
@@ -128,11 +128,11 @@ func promoteRunLocalOutput(
|
|||||||
return artifacts.Ref{}, fmt.Errorf("read run-local output %q: %w", srcPath, err)
|
return artifacts.Ref{}, fmt.Errorf("read run-local output %q: %w", srcPath, err)
|
||||||
}
|
}
|
||||||
if err := store.WriteFileAtomic(canonicalPath, data, 0o644); err != nil {
|
if err := store.WriteFileAtomic(canonicalPath, data, 0o644); err != nil {
|
||||||
return artifacts.Ref{}, fmt.Errorf("promote output to %q: %w", canonicalPath, err)
|
return artifacts.Ref{}, fmt.Errorf("materialize output to %q: %w", canonicalPath, err)
|
||||||
}
|
}
|
||||||
checksum, err := store.Checksum(canonicalPath)
|
checksum, err := store.Checksum(canonicalPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return artifacts.Ref{}, fmt.Errorf("checksum promoted output %q: %w", canonicalPath, err)
|
return artifacts.Ref{}, fmt.Errorf("checksum materialized output %q: %w", canonicalPath, err)
|
||||||
}
|
}
|
||||||
ref.AbsolutePath = canonicalPath
|
ref.AbsolutePath = canonicalPath
|
||||||
ref.Checksum = checksum
|
ref.Checksum = checksum
|
||||||
|
|||||||
@@ -213,11 +213,11 @@ dispatch:
|
|||||||
ref := outputRef[speaker]
|
ref := outputRef[speaker]
|
||||||
runOutputPaths = append(runOutputPaths, ref.AbsolutePath)
|
runOutputPaths = append(runOutputPaths, ref.AbsolutePath)
|
||||||
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
|
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
|
||||||
promoted, err := promoteRunLocalOutput(env.ArtifactStore, ref.AbsolutePath, canonicalOut, ref)
|
materialized, err := materializeRunLocalOutput(env.ArtifactStore, ref.AbsolutePath, canonicalOut, ref)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("transcribe: promote %q output: %w", speaker, err)
|
return nil, fmt.Errorf("transcribe: materialize %q output: %w", speaker, err)
|
||||||
}
|
}
|
||||||
outputs = append(outputs, promoted)
|
outputs = append(outputs, materialized)
|
||||||
outputPaths = append(outputPaths, canonicalOut)
|
outputPaths = append(outputPaths, canonicalOut)
|
||||||
orderedPerFile[speaker] = perFile[speaker]
|
orderedPerFile[speaker] = perFile[speaker]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -217,7 +217,7 @@ func TestTranscribeStageUsesRunLocalOutputAndPromotesCanonical(t *testing.T) {
|
|||||||
t.Fatalf("outputs = %#v, want one output", result.Outputs)
|
t.Fatalf("outputs = %#v, want one output", result.Outputs)
|
||||||
}
|
}
|
||||||
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
|
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
|
||||||
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
|
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
|
||||||
}
|
}
|
||||||
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
materializedTrimmed, err := materializeRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
||||||
Kind: "transcript_final_trimmed",
|
Kind: "transcript_final_trimmed",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -110,7 +110,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
}
|
}
|
||||||
metadata["trim_action"] = "copy_disabled"
|
metadata["trim_action"] = "copy_disabled"
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: []artifacts.Ref{promotedTrimmed},
|
Outputs: []artifacts.Ref{materializedTrimmed},
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -351,7 +351,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
return nil, fmt.Errorf("trim: trimmed transcript %q invalid: %w", trimmedPath, err)
|
return nil, fmt.Errorf("trim: trimmed transcript %q invalid: %w", trimmedPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
materializedTrimmed, err := materializeRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
||||||
Kind: "transcript_final_trimmed",
|
Kind: "transcript_final_trimmed",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -359,7 +359,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
|
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
|
||||||
}
|
}
|
||||||
promotedBounds, err := promoteRunLocalOutput(env.ArtifactStore, finalBoundsOutputPath, canonicalBoundsOutputPath, artifacts.Ref{
|
materializedBounds, err := materializeRunLocalOutput(env.ArtifactStore, finalBoundsOutputPath, canonicalBoundsOutputPath, artifacts.Ref{
|
||||||
Kind: "session_bounds",
|
Kind: "session_bounds",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
@@ -369,7 +369,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: []artifacts.Ref{promotedTrimmed, promotedBounds},
|
Outputs: []artifacts.Ref{materializedTrimmed, materializedBounds},
|
||||||
Logs: logPaths,
|
Logs: logPaths,
|
||||||
GeneratedConfigs: generatedConfigs,
|
GeneratedConfigs: generatedConfigs,
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
|
|||||||
@@ -326,7 +326,7 @@ func TestTrimStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, out := range result.Outputs {
|
for _, out := range result.Outputs {
|
||||||
if strings.Contains(out.AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
if strings.Contains(out.AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
t.Fatalf("promoted output path = %q, want canonical session path", out.AbsolutePath)
|
t.Fatalf("materialized output path = %q, want canonical session path", out.AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user