30 Commits

Author SHA1 Message Date
37daab7857 Bugfix involving nested directory creation
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-18 11:52:40 +00:00
2356688cb9 Removed legacy interfaces and old documentation references to the previous on-disk layout 2026-05-18 03:02:22 +00:00
1054b64d9f Implement minimal downstream invalidation after forced upstream reruns 2026-05-18 01:50:51 +00:00
01fb02426c Update the analyze stage to utilize the new artifact package 2026-05-18 01:29:18 +00:00
7dc79e052f Aligned the archive stage with the new work directory layout 2026-05-18 01:13:51 +00:00
cb525c0f72 Implemented run-local stage execution + immediate promotion for core output-producing stages 2026-05-18 00:55:35 +00:00
622677d038 Added run manifest scaffolding and helpers 2026-05-17 21:15:51 +00:00
550288e008 Add campaign-aware workspace path foundation 2026-05-17 20:57:27 +00:00
e58e545686 Audit workspace architecture implementation plan
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-17 13:22:09 -05:00
6ff54c5a0f Documentation update and reorganization 2026-05-17 13:14:28 -05:00
924b5d15c6 Applied a more general bugfix to path-resolution issues in the archive stage 2026-05-17 11:08:11 -05:00
b065663180 Bugfix involving path resolution in the archive stage 2026-05-17 11:03:02 -05:00
3ba564b00f Bugfix involving directory creation during the merge stage 2026-05-17 08:18:57 -05:00
a3986cf0d6 Centralized defaults into internal/config/defaults.go 2026-05-17 07:53:51 -05:00
539601bd16 Updated the merge stage to normalize the per-speaker transcripts before merging them
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 23:30:13 -05:00
6ca1c8d6b0 The backend S3 client now resolves credentials from user-configurable environment variables 2026-05-16 23:22:21 -05:00
4b7b50981b Add .gocache to .gitignore and minor documentation cleanup 2026-05-16 23:21:45 -05:00
33f7ae8f2e Simplify downstream tool configuration
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 23:30:40 +00:00
d5a9ad38f8 Add post-archive cleanup policies 2026-05-16 23:09:39 +00:00
6fbefb9867 Add session discovery and template support 2026-05-16 22:57:42 +00:00
1665359486 Added a locally generated UX progress report 2026-05-16 20:11:20 +00:00
03f2543927 Created a UX status report 2026-05-16 20:09:53 +00:00
fe9c348092 Document and review S3 archive workflow 2026-05-16 15:24:44 +00:00
f7f8f1a949 Promote current session artifacts to storage 2026-05-16 15:01:01 +00:00
d40c91acde Upload successful run records to storage 2026-05-16 14:43:29 +00:00
ed4dcf1ef7 Updated go.mod 2026-05-16 09:34:43 -05:00
24cce49a70 Download S3 audio during prepare 2026-05-16 14:33:42 +00:00
1e6db89dd4 Add remote storage backend 2026-05-16 14:22:04 +00:00
0454296c81 Add archive storage path configuration 2026-05-16 14:11:59 +00:00
58c6ab2d54 Updated audita configuration to reflect the new audita public CLI
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 08:46:48 -05:00
93 changed files with 9269 additions and 1865 deletions

5
.gitignore vendored
View File

@@ -2,6 +2,8 @@
.codex
AGENTS.md
.DS_Store
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -22,6 +24,9 @@ AGENTS.md
# Dependency directories (remove the comment below to include it)
# vendor/
# Go cache
.gocache
# Go workspace file
go.work
go.work.sum

177
README.md
View File

@@ -16,7 +16,6 @@ Implemented now:
Not implemented yet:
- `archive` stage behavior
- `notify` stage behavior
- additional analyze artifacts beyond `session_recap`
- generic DAG orchestration
@@ -35,6 +34,25 @@ Pipeline config lookup for CLI commands:
- `/usr/local/etc/narratio/pipeline.yml`
- `/etc/narratio/pipeline.yml`
Session config lookup for CLI commands:
- if `--session <path>` is provided, Narratio uses that path
- if `--session` is omitted, Narratio searches in this order:
- `./session.yml`
- `/usr/local/etc/narratio/session.yml`
- `/etc/narratio/session.yml`
Session template support:
- Narratio renders `session.yml` templates before strict YAML decode.
- `--session-id <value>` provides the `session_id` template variable.
- Supported placeholder forms:
- `{{session_id}}`
- `{{ session_id }}`
- unresolved template placeholders fail with a clear error.
- strict YAML validation still runs after rendering.
- concrete `session.yml` files without templates remain fully supported.
Optional secrets-from-files config:
- `pipeline.secrets.env_dir` may point to a directory of secret files
@@ -47,6 +65,89 @@ Optional secrets-from-files config:
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
Maintainer note: application defaults are centralized in [`internal/config/defaults.go`](internal/config/defaults.go).
## Storage And Archive Foundations
Narratio now includes configuration and path-model foundations for archive support, plus implemented prepare-stage S3 audio input.
Implemented foundations:
- `pipeline.storage.s3` config shape (`bucket`, `root_prefix`, `region`, `endpoint`, `force_path_style`, `access_key_id_env`, `secret_access_key_env`)
- `pipeline.spool` config shape (`root`, `delete_audio_after_archive`)
- `pipeline.archive` config shape (`enabled`, `upload_run`, `promote_artifacts`)
- promotion-rule validation (`from`/`to` required, relative-only paths, traversal rejected)
- `session.campaign` requirement for campaign-aware path construction
- optional `session.inputs.audio_s3.prefix` modeling and prepare-stage S3 audio download
- run ID generation and S3/local path helper foundations
- manifest run/path identity fields
Current defaults:
- `pipeline.storage.s3.root_prefix`: `dnd`
- `pipeline.storage.s3.access_key_id_env`: `OBJECT_STORAGE_KEY_ID`
- `pipeline.storage.s3.secret_access_key_env`: `OBJECT_STORAGE_KEY`
- `pipeline.workspace.cleanup_after_archive`: `false`
- `pipeline.spool.root`: `/var/spool/narratio`
- `pipeline.spool.delete_audio_after_archive`: `false`
- `pipeline.archive.enabled`: `true`
- `pipeline.archive.upload_run`: `true`
- default `pipeline.archive.promote_artifacts`:
- `transcripts/trimmed.json` -> `transcripts/trimmed.json` (`required: true`)
- `artifacts/session_recap.md` -> `artifacts/session_recap.md` (`required: true`)
Current boundaries:
- local development audio (`audio_dir` / `audio_files`) still works
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
- real S3-compatible backend now exists in the storage adapter package
- storage backend tests use fake storage and do not require live S3
- archive uploads successful run records under `runs/{run_id}/`
- archive does not upload local audio by default
- archive uploads promoted outputs to session-level keys using `archive.promote_artifacts`
- archive uploads `current/manifest.json`
- archive uploads `current/run_id.txt` last as the effective commit marker
- required missing promotions fail archive
- optional missing promotions are skipped and recorded
- cleanup remains conservative and opt-in:
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory after successful archive commit
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir after successful archive commit
- cleanup executes only after all selected stages for the command invocation succeed
- cleanup does not run for failed, incomplete, skipped, or unarchived runs
- local development `audio_dir`/`audio_files` source inputs are never deleted by spool cleanup
- S3 credentials are resolved from configured env-var names when both are present; if either is missing, Narratio falls back to the AWS SDK default credential chain
S3 input details and current boundaries are documented in [docs/s3-audio-input.md](docs/s3-audio-input.md).
## Remote Storage Backend
Narratio includes an object-store backend layer for future prepare/archive work:
- `List(ctx, prefix)`
- `Download(ctx, key, localPath)`
- `Upload(ctx, localPath, key, opts)`
- `Exists(ctx, key)`
Implemented backends:
- fake storage backend for deterministic tests
- S3-compatible backend built from `pipeline.storage.s3`
Key invariant:
- callers pass full bucket-relative object keys
- storage backends do not prepend `root_prefix` and do not infer session/campaign paths
Current boundary:
- `prepare` uses `List` + `Download` through the backend when `session.inputs.audio_s3` is configured
- `archive` uses `Upload` through the backend for successful run-record uploads
- `archive` also uses `Upload` for promotion writes and current pointers
- no failed or incomplete runs are uploaded
- local audio is not re-uploaded by default
Archive run-upload details and boundaries are documented in [docs/archive-storage.md](docs/archive-storage.md).
## Canonical Stage Order
1. `prepare`
@@ -66,6 +167,52 @@ YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
## Seriatim Configuration
`pipeline.seriatim` configures the Seriatim subprocess adapter used by `merge`, `normalize`, and `trim`.
Minimal behavior:
- `pipeline.seriatim` may be omitted entirely.
- when omitted, Narratio defaults to:
- `binary: seriatim`
- `timeout: 10m`
- `output_schema: seriatim-intermediate`
- `coalesce_gap: 3.0`
- `report: true`
Optional overrides in `pipeline.seriatim` continue to work, including explicit binary paths and advanced `env` tuning values.
## Audita Configuration
`pipeline.audita` configures the real Audita subprocess adapter used by `polish`.
Minimal behavior:
- `pipeline.audita` may be omitted entirely.
- when omitted, Narratio defaults to:
- `binary: audita`
- `timeout: 3h`
- `report: true`
Optional:
- `llm_api_key_env` (when set, Narratio requires that env var and passes it to Audita as `AUDITA_LLM_API_KEY`)
- `modules` override list (when empty/omitted, Narratio does not pass `--modules`)
- `base_url` (when omitted, Narratio does not pass `--base-url`; Audita runtime defaults/config may apply)
- `model` (when omitted, Narratio does not pass `--model`; Audita runtime defaults/config may apply)
- `transcript_description`
- `config_path`
- `output_schema` (`bare-segments` or `audita-v1`)
- `work_dir_retention` (`always`, `auto`, or `never`)
- `total_llm_concurrency` (> 0 when provided)
- `proposal_llm_concurrency` (> 0 when provided)
- `validation_model`
- `validation_llm_concurrency` (> 0 when provided)
- `report` (defaults to `true`)
Narratio passes only configured optional Audita flags. Omitted optional values are left to Audita runtime defaults/config.
## Normalize Configuration
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
@@ -159,7 +306,7 @@ Render-debug files are diagnostics and are not treated as canonical stage output
Key points:
- `scriptorium.binary` is required when section is present
- `scriptorium.binary` defaults to `scriptorium` when section is present
- `scriptorium.config_path` is optional
- `scriptorium.timeout` defaults to `10m` when omitted
- `scriptorium.render_debug` enables render diagnostics globally
@@ -222,10 +369,16 @@ For the initial implementation, only `session_recap` generation is supported.
Analyze-stage session recap behavior:
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
- preferred transcript artifact source IDs:
- `narratio.transcript.polished`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
- backward-compatible aliases remain supported:
- `processed_transcript`
- `normalized_transcript`
- `trimmed_transcript`
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
- Narratio resolves transcript inputs from the artifact resolver (manifest producer outputs first, then canonical session paths)
- missing trimmed transcript fails clearly and advises running trim stage first
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
- `processed_transcript` remains supported for advanced/debug use cases
@@ -247,7 +400,9 @@ Expected session output paths:
Starter files:
- `examples/pipeline.minimal.yml`
- `examples/pipeline.audita-overrides.yml`
- `examples/session.minimal.yml`
- `examples/session.template.yml`
- `examples/speakers.yml`
## Commands
@@ -266,6 +421,12 @@ go run ./cmd/narratio plan --session examples/session.minimal.yml
Use `--config <path>` to override default pipeline lookup when needed.
Run with a discoverable session template:
```bash
go run ./cmd/narratio run --session-id 2026-04-04
```
Run full pipeline:
```bash
@@ -278,6 +439,12 @@ Run analyze only:
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
```
Resume with a template session ID:
```bash
go run ./cmd/narratio resume --config examples/pipeline.minimal.yml --session examples/session.template.yml --session-id 2026-04-04
```
## Operational Note
Checksum-based stale detection is not implemented yet.

View File

@@ -22,10 +22,24 @@ Implemented:
- real `trim` stage producing `transcripts/trimmed.json`
- real `analyze` stage for initial `session_recap` generation
- optional Scriptorium render diagnostics (`render_debug`) before production run
- storage/archive configuration and validation foundations for:
- `pipeline.storage.s3`
- `pipeline.spool`
- `pipeline.archive` promotion rules
- `session.inputs.audio_s3`
- run identity and path-model foundations:
- run ID generation (`YYYYMMDDTHHMMSSZ-xxxxxxxx`)
- S3 session/run/current key builders
- campaign/session/run local work/spool path helpers
- manifest run/path identity fields (`campaign`, `run_id`, local and S3 prefixes)
- remote storage backend layer:
- narrow object-store interface (`List`, `Download`, `Upload`, `Exists`)
- fake storage backend for deterministic tests (no network dependency)
- S3-compatible backend using AWS SDK v2
- config-based object-store construction helper
Still placeholder/future:
- `archive` stage behavior
- `notify` stage behavior
- additional Scriptorium artifact types beyond `session_recap`
- artifact-to-artifact workflows beyond the initial single-artifact implementation
@@ -100,6 +114,26 @@ CLI pipeline config path resolution:
- when `--config` is omitted, Narratio searches defaults in order:
- `/usr/local/etc/narratio/pipeline.yml`
- `/etc/narratio/pipeline.yml`
- default values are centralized in `internal/config/defaults.go`
CLI session config path resolution:
- when `--session <path>` is provided, that path is used
- when `--session` is omitted, Narratio searches defaults in order:
- `./session.yml`
- `/usr/local/etc/narratio/session.yml`
- `/etc/narratio/session.yml`
Session template rendering:
- session templates are rendered before strict YAML decode
- `--session-id <value>` provides the `session_id` template variable
- supported placeholders:
- `{{session_id}}`
- `{{ session_id }}`
- unresolved placeholders fail clearly
- strict `KnownFields(true)` YAML validation still applies after rendering
- if rendered `session.session_id` conflicts with `--session-id`, load fails clearly
Optional pipeline secrets directory:
@@ -110,12 +144,147 @@ Optional pipeline secrets directory:
- if configured, unreadable/missing `env_dir` fails command execution early
- relative `env_dir` values are resolved from current working directory
Storage and archive foundations:
- `pipeline.storage.s3` is available for modeling S3 coordinates:
- `bucket`
- `root_prefix` (default `dnd`)
- `region`
- `endpoint`
- `force_path_style` (default `false`)
- `access_key_id_env` (default `OBJECT_STORAGE_KEY_ID`)
- `secret_access_key_env` (default `OBJECT_STORAGE_KEY`)
- `pipeline.spool.root` defaults to `/var/spool/narratio`
- `pipeline.workspace.cleanup_after_archive` defaults to `false`
- `pipeline.spool.delete_audio_after_archive` defaults to `false`
- `pipeline.archive` is optional and defaults to:
- `enabled: true`
- `upload_run: true`
- default `promote_artifacts`:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
- archive promotion rules enforce safe relative paths:
- `from` and `to` are required
- absolute paths are rejected
- traversal segments such as `..` are rejected
Session input foundations:
- `session.campaign` is required
- local audio remains supported through `session.inputs.audio_dir` or `session.inputs.audio_files`
- optional S3 audio input shape is `session.inputs.audio_s3.prefix`
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
- when `audio_s3` is configured, `prepare` lists and downloads `.flac` objects through the object-store backend
Cross-config validation scope:
- `pipeline.storage.s3.bucket` is required only when an S3-dependent feature is explicitly configured (for current foundations, that includes `session.inputs.audio_s3`, and archive upload intent when using `storage.backend: s3`)
- no AWS credential values are stored in Narratio config; only env-var names are configured
- when both configured credential env vars resolve to non-empty values, the S3 backend uses them as static credentials
- when either configured credential value is missing, the S3 backend falls back to the AWS SDK default credential chain
Remote object-store backend scope:
- remote storage APIs are isolated to `internal/adapters/storage`
- AWS SDK types remain contained within the S3 backend implementation package
- S3 key/session path semantics remain outside the backend, with this invariant:
- callers pass full bucket-relative object keys
- backend methods do not prepend `root_prefix` or infer campaign/session/run paths
- `prepare` now uses object-store `List` and `Download` for S3 audio input
- `archive` now uses object-store `Upload` for successful run-record upload under the run prefix
- `archive` now uses object-store `Upload` for promoted outputs and current pointers
Prepare S3 audio behavior (implemented):
- compute session prefix as `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
- resolve `session.inputs.audio_s3.prefix` under that session prefix
- list objects under the computed audio prefix and filter `.flac` keys
- fail clearly when no `.flac` objects are found
- download selected objects to spool audio path:
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
- materialize audio files into workdir audio path:
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
- record S3 provenance in manifest input records (bucket/key/metadata/local paths/checksum)
- no AWS SDK types are used in stage code; storage implementation details stay in storage adapter packages
Archive publishing behavior (implemented):
- `archive` verifies prerequisite stage success before upload:
- `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`
- only successful/completed runs are uploaded
- uploaded run record destination is:
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
- uploaded existing local paths include:
- `inputs/`, `transcripts/`, `artifacts/`, optional `reports/`, `config/`, `logs/`, and `manifest.json`
- local `audio/` is intentionally excluded from upload by default
- file upload order is deterministic (sorted relative paths)
- `archive.enabled: false` and `archive.upload_run: false` skip upload cleanly
- stage metadata records non-secret upload context:
- run upload details, promoted output details, current manifest key, current pointer key
- no secrets, transcript contents, prompt contents, or environment dumps
- promotion rules:
- `from` resolves from local workdir
- `to` resolves under session-level S3 root
- missing required source fails archive
- missing optional source is skipped and recorded
- default promoted outputs:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
- current pointers:
- `current/manifest.json` uploaded after run upload and promotions
- `current/run_id.txt` uploaded last with `{run_id}\n`
- `current/run_id.txt` is the effective commit marker
- if promotion or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`
- failed/incomplete runs remain local and are not uploaded
- post-archive local cleanup (implemented, opt-in):
- cleanup runs only after archive succeeded and wrote `current/run_id.txt`
- cleanup is executed after all selected stages in the command invocation succeed (for example, a later `notify` failure leaves local files intact)
- `pipeline.spool.delete_audio_after_archive: true` removes only `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
- `pipeline.workspace.cleanup_after_archive: true` removes only `{workspace.root}/work/{campaign}/{session_id}/{run_id}/`
- cleanup does not run when archive is skipped/disabled/fails or when run upload is disabled
- local development `audio_dir`/`audio_files` inputs are never removed by spool cleanup
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
`pipeline.audita` drives the real Audita subprocess adapter for the `polish` stage.
Audita defaulted fields:
- `binary` defaults to `audita`
- `timeout` defaults to `3h`
- `report` defaults to `true`
Audita optional fields:
- `llm_api_key_env` (enforced only when configured)
- `modules` override list (when omitted/empty, Narratio does not pass `--modules`)
- `base_url` (when omitted, Narratio does not pass `--base-url`)
- `model` (when omitted, Narratio does not pass `--model`)
- `transcript_description`
- `config_path`
- `output_schema` (`bare-segments` or `audita-v1`)
- `work_dir_retention` (`always`, `auto`, `never`)
- `total_llm_concurrency` (> 0 when provided)
- `proposal_llm_concurrency` (> 0 when provided)
- `validation_model`
- `validation_llm_concurrency` (> 0 when provided)
- `report` override
Narratio passes only configured optional Audita flags; omitted optional values defer to Audita runtime defaults/config.
Seriatim defaults:
- `pipeline.seriatim` may be omitted
- `binary` defaults to `seriatim`
- `timeout` defaults to `10m`
- `output_schema` defaults to `seriatim-intermediate`
- `coalesce_gap` defaults to `3.0`
- `report` defaults to `true`
When `pipeline.normalize` is omitted, defaults are applied:
- `output_path: transcripts/normalized.json`
@@ -148,7 +317,7 @@ When `pipeline.trim.enabled: true`:
When `pipeline.scriptorium` is present:
- `binary` is required and non-empty
- `binary` defaults to `scriptorium` when omitted
- `config_path` is optional; when provided it must be non-empty
- `timeout` is optional; when provided it must parse as a Go duration
- default `timeout` is `10m`

View File

@@ -0,0 +1,130 @@
# Workspace Architecture Implementation Plan (Status)
This document tracks the implemented workspace architecture and remaining work for v1.0.
## Current Architecture (Implemented)
Narratio now uses a canonical campaign-aware local layout:
```text
{workspace.root}/work/{campaign_id}/{session_id}/
manifest.json
current/
manifest.json
run_id.txt
inputs/
transcripts/
artifacts/
reports/
logs/
config/
runs/
{run_id}/
manifest.json
{stage}/
outputs/
logs/
reports/
config/
scratch/
```
Core behavior:
- Session manifest remains the skip/resume source of truth.
- Each invocation creates a run manifest at `runs/{run_id}/manifest.json`.
- Stage execution writes run-local artifacts and promotes durable outputs to canonical session paths.
- Archive uploads run records under `runs/{run_id}/`, applies promotion rules, then publishes `current/manifest.json` and `current/run_id.txt`.
- Analyze input resolution uses centralized artifact IDs with alias support.
- Forced upstream reruns mark downstream succeeded stages `stale` so later runs do not skip stale outputs.
## Section 4 Sequence Status
### Step 1: Campaign-aware session path model
Status: complete.
Implemented:
- Campaign-aware session and run path helpers.
- Campaign-aware artifact-store layout APIs.
- Canonical session manifest pathing under `work/{campaign}/{session}`.
### Step 2: Session manifest + run manifest scaffolding
Status: complete.
Implemented:
- Invocation-scoped run manifest type and store methods.
- Runner creates/saves run manifests per invocation.
- Session manifest remains authoritative for idempotent stage skipping.
### Step 3: Run-local stage execution + promotion
Status: complete.
Implemented:
- Run-local stage directory layout under `runs/{run_id}/{stage}`.
- Shared helpers for run-local output mapping and promotion to canonical durable paths.
- Producer run provenance recorded on durable artifact outputs.
### Step 4: Archive alignment
Status: complete.
Implemented:
- Canonical run-root/session-root resolution.
- Deterministic run-file collection and promotion source resolution.
- Current-pointer publication ordering retained (`current/manifest.json` then `current/run_id.txt`).
### Step 5: Artifact registry/resolver (analyze first consumer)
Status: complete.
Implemented:
- Central artifact resolver with canonical IDs:
- `narratio.transcript.merged`
- `narratio.transcript.polished`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
- `narratio.bounds.session`
- `narratio.artifact.session_recap`
- Backward-compatible aliases:
- `processed_transcript`
- `normalized_transcript`
- `trimmed_transcript`
- Analyze stage switched to resolver-based source resolution.
### Step 6: Minimal downstream invalidation for forced reruns
Status: complete.
Implemented:
- Deterministic downstream invalidation based on canonical stage order.
- On forced successful rerun of stage `X`, downstream succeeded stages are marked `stale`.
- Resume and non-forced runs naturally re-execute stale stages.
### Step 7: Legacy layout migration strategy
Status: intentionally skipped.
Decision:
- Automatic migration and legacy fallback compatibility are intentionally not implemented.
- The codebase targets canonical-only local layout behavior.
- Legacy local workspace state, if present, should be recreated or migrated manually outside Narratio.
## Remaining Work (v1.0)
No required workspace/run-history migration steps remain from Section 4.
Possible future enhancements (non-blocking):
- Full checksum/input-graph stale detection.
- Optional retention-policy expansion for run-history cleanup.
- Broader artifact-resolver adoption across additional stage consumers.

View File

@@ -0,0 +1,744 @@
# Narratio Workspace, Run History, and Artifact Resolution Architecture
## 1. Purpose
This document defines the intended v1.0 architecture for Narratio's local workspace layout, run history model, durable session outputs, manifest responsibilities, and artifact resolution contract.
Narratio is an idempotent session orchestrator. The command:
```bash
narratio run --session-id 2026-05-07
```
means "bring the identified session to its desired completed state." It does **not** mean "always create an entirely new independent output tree and ignore prior session state."
This distinction drives the architecture:
* A **session** is the durable domain object and idempotency boundary.
* A **run** is an execution attempt that may update the session's durable state.
* Durable outputs live at the session level.
* Run-specific outputs, logs, generated configs, scratch files, and diagnostics live under `runs/{run_id}/`.
* Successful stage outputs are promoted from run-local locations into canonical session-level locations.
* The session manifest records current durable state.
* Run manifests record execution history and debugging/provenance details.
This model intentionally mirrors the S3 archive model: session-level current artifacts are distinct from run-record history.
## 2. Core Concepts
### 2.1 Session
A session is the stable unit of work identified by `campaign_id` and `session_id`.
Examples:
```text
campaign_id = dilfs
session_id = 2026-05-07
```
The session directory represents the current durable local state for that session. Re-running Narratio for the same session should consult this state, skip already-completed stages by default, and produce no changes unless work is incomplete, stale, forced, or explicitly selected.
### 2.2 Run
A run is a particular execution attempt identified by a generated `run_id`, for example:
```text
20260517T174748Z-abcd1234
```
A run may execute all stages or only a sparse subset of stages. Sparse runs are expected and desirable when the user invokes `--force`, `run-stage`, or a stage-limited command.
Run directories are provenance/debug records. They should reflect what actually happened during that invocation, not a synthetic complete pipeline layout.
### 2.3 Durable Output
A durable output is a canonical session-level artifact intended for later stages, user consumption, archive promotion, or future idempotency decisions.
Examples:
```text
transcripts/merged.json
transcripts/processed.json
transcripts/normalized.json
transcripts/trimmed.json
artifacts/session_recap.md
```
Durable outputs live directly under the session directory, not under a particular run directory.
### 2.4 Run-Local Output
A run-local output is the file initially produced by a stage during a specific run. After validation, durable outputs are promoted from run-local paths to session-level canonical paths.
Run-local outputs, logs, generated configs, reports, and scratch files should remain under:
```text
runs/{run_id}/{stage}/...
```
## 3. Local Workspace Layout
The canonical local workspace layout is:
```text
{workspace.root}/work/{campaign_id}/{session_id}/
manifest.json
current/
manifest.json
run_id.txt
inputs/
transcripts/
artifacts/
reports/
logs/
config/
runs/
{run_id}/
manifest.json
prepare/
transcribe/
merge/
polish/
normalize/
trim/
analyze/
archive/
notify/
```
Not every directory must exist at all times. Directories should be created idempotently when needed.
### 3.1 Session Root
The session root is:
```text
{workspace.root}/work/{campaign_id}/{session_id}/
```
The session root is the stable local home for the session. It is the default base for resolving canonical artifact paths.
The only files that should live directly in the session root are core session-state files, primarily:
```text
manifest.json
```
Lock files may also be session-root scoped if the implementation uses file locks there, but transient locks should not be treated as durable artifacts.
### 3.2 Session-Level Canonical Directories
The following directories contain current durable session state:
```text
inputs/
transcripts/
artifacts/
reports/
logs/
config/
current/
```
Recommended meanings:
| Directory | Purpose |
| -------------- | ----------------------------------------------------------------------------- |
| `inputs/` | Materialized or copied input files used by the current durable session state. |
| `transcripts/` | Canonical transcript tiers. |
| `artifacts/` | User-facing and machine-readable generated artifacts. |
| `reports/` | Canonical stage reports worth preserving at the session level. |
| `logs/` | Optional session-level logs or promoted/latest logs. |
| `config/` | Optional session-level generated config snapshots or promoted/latest configs. |
| `current/` | Current published session pointers, mirroring the archive backend. |
Canonical durable outputs should use stable paths under these directories.
### 3.3 Run History Directory
Run history lives under:
```text
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/
```
Each run directory records what happened during that invocation. A run may contain all stage directories or only a sparse subset.
Example full run:
```text
runs/20260517T174748Z-abcd1234/
manifest.json
prepare/
transcribe/
merge/
polish/
normalize/
trim/
analyze/
archive/
notify/
```
Example sparse forced analyze run:
```text
runs/20260518T030000Z-efgh5678/
manifest.json
analyze/
```
Example sparse polish-through-analyze rerun:
```text
runs/20260518T041500Z-a1b2c3d4/
manifest.json
polish/
normalize/
trim/
analyze/
```
Run directories should not create stage folders for stages that were not selected, executed, skipped, or otherwise considered during that run unless there is a clear diagnostic reason to do so.
### 3.4 Stage Run-Local Directories
Each stage receives a run-local directory:
```text
runs/{run_id}/{stage}/
```
Within that stage directory, the stage may use subdirectories such as:
```text
outputs/
logs/
reports/
config/
scratch/
```
For example:
```text
runs/{run_id}/polish/
outputs/transcripts/processed.json
reports/audita.polish.report.json
logs/stdout.log
logs/stderr.log
config/audita.polish.generated.yml
scratch/
```
The exact internal layout of a stage directory may vary by stage, but it should be deterministic, documented, and generated through centralized path helpers rather than ad hoc path joins.
## 4. Promotion Model
Narratio uses stage-level promotion with immediate promotion after successful validation.
The stage lifecycle is:
1. Resolve required inputs from the current session state and/or run-local context.
2. Create the run-local stage directory.
3. Execute the stage, writing outputs under `runs/{run_id}/{stage}/...`.
4. Validate run-local outputs.
5. Promote durable outputs into session-level canonical paths.
6. Update the session manifest.
7. Update the run manifest.
Promotion means an atomic or effectively atomic copy/rename from a run-local path to a session-level canonical path.
Example:
```text
runs/{run_id}/polish/outputs/transcripts/processed.json
```
is promoted to:
```text
transcripts/processed.json
```
Promotion should be safe and deterministic:
* Validate before promotion.
* Write promoted files atomically where possible.
* Never leave partially written durable outputs.
* Record the producing `run_id` in the session manifest.
* Preserve run-local files for debugging unless retention policy deletes them.
## 5. Promotion Policy: Option A
Narratio uses immediate stage-level promotion.
If a selected stage succeeds, its durable outputs are promoted immediately, even if a later selected stage fails.
Example:
```bash
narratio run --session-id 2026-05-07 --force --stages polish,normalize,trim,analyze
```
If `polish` succeeds and `normalize` fails:
* `transcripts/processed.json` may be updated from the new run.
* `normalize`, `trim`, and `analyze` should not be marked succeeded for the new input state.
* Downstream outputs may now be stale relative to the newly promoted polished transcript.
This policy is simpler, transparent, and consistent with stage-level resumability. It does require explicit stale/invalidation handling.
## 6. Stale and Invalidation Semantics
Full checksum-based stale detection may be implemented later. Before that exists, Narratio should still use a simple deterministic invalidation rule for forced or explicit upstream reruns.
When a stage is successfully re-executed and promoted, downstream stages should be marked stale unless they are also re-executed successfully in the same command invocation.
Example stage order:
```text
prepare -> transcribe -> merge -> polish -> normalize -> trim -> analyze -> archive -> notify
```
If `polish` is forced and promoted, then the following downstream stages should be invalidated unless rerun successfully:
```text
normalize
trim
analyze
archive
notify
```
A stale stage is not equivalent to a failed stage. It means its current durable outputs may no longer correspond to current upstream inputs or configuration.
Minimum manifest state model:
```text
pending
running
succeeded
failed
skipped
stale
```
If adding a new `stale` state is too invasive for v1.0, the implementation should at least record stale metadata or clear downstream success markers in a way that prevents accidental idempotent skips based on obsolete outputs.
## 7. Manifest Responsibilities
Narratio should distinguish between session manifests and run manifests.
The same underlying Go types may be reused where practical, but the concepts should remain separate.
### 7.1 Session Manifest
Path:
```text
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
```
The session manifest answers:
```text
What is the current durable state of this session?
```
It should record:
* campaign ID
* session ID
* current or latest run ID
* current stage states
* canonical durable output refs
* artifact IDs and paths
* producing run ID for each current stage output
* relevant input/config checksums when available
* stale/invalidated stage information
* archive/current publication metadata
A session's durable state may be a composite of multiple runs.
For example:
```text
transcripts/merged.json produced by run A
transcripts/processed.json produced by run B
transcripts/normalized.json produced by run B
transcripts/trimmed.json produced by run B
artifacts/session_recap.md produced by run C
```
This is valid and expected.
### 7.2 Run Manifest
Path:
```text
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/manifest.json
```
The run manifest answers:
```text
What happened during this specific execution attempt?
```
It should record:
* run ID
* campaign ID
* session ID
* command mode and selected stages
* force flags or stage selection flags
* stages considered during this run
* stages executed during this run
* stages skipped during this run and reasons
* run-local output paths
* promoted output paths
* logs
* reports
* generated configs
* timings
* errors
* non-secret subprocess invocation metadata
Run manifests are primarily for debugging, auditability, and archive history.
## 8. Idempotency and Resume Behavior
The idempotency boundary is the session, not the run.
By default:
```bash
narratio run --session-id 2026-05-07
```
should consult the session manifest and skip stages that are already succeeded and not stale.
If all stages are already complete, the command should execute zero stages and report that the session is already complete.
Forced execution creates a new run record but updates session-level durable state only for stages that actually succeed and promote outputs.
Examples:
```bash
narratio run --session-id 2026-05-07 --force
```
Creates a new run and attempts to re-execute the selected/default stage set.
```bash
narratio run-stage --session-id 2026-05-07 analyze --force
```
Creates a sparse run that executes only `analyze`, then promotes updated analysis artifacts if successful.
```bash
narratio resume --session-id 2026-05-07
```
Uses the session manifest to determine what remains incomplete or stale. Resume does not need to resume the same `run_id` unless the implementation explicitly supports resuming an interrupted active run.
## 9. Artifact Resolution Contract
Narratio should provide a first-class artifact registry and resolver.
The resolver maps symbolic artifact source names to canonical session-level paths and manifest output kinds.
Stages and adapters should not hardcode path fragments when resolving cross-stage inputs. They should ask the artifact resolver for the current durable artifact by ID.
### 9.1 Canonical Artifact IDs
Preferred artifact IDs should be namespaced:
```text
narratio.transcript.merged
narratio.transcript.polished
narratio.transcript.full
narratio.transcript.trimmed
narratio.bounds.session
narratio.artifact.session_recap
```
Recommended initial registry:
| Artifact ID | Canonical Path | Producer Stage | Output Kind | Meaning |
| --------------------------------- | ------------------------------- | -------------- | ------------------------ | ------------------------------------- |
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` | Deterministic Seriatim merge. |
| `narratio.transcript.polished` | `transcripts/processed.json` | `polish` | `transcript_processed` | Full Audita-polished transcript. |
| `narratio.transcript.full` | `transcripts/normalized.json` | `normalize` | `transcript_normalized` | Preferred full normalized transcript. |
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` | Gameplay-only transcript. |
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` | Trim bounds selected for the session. |
| `narratio.artifact.session_recap` | `artifacts/session_recap.md` | `analyze` | `artifact_session_recap` | Generated session recap. |
### 9.2 Backward-Compatible Aliases
Existing source names should remain supported:
| Legacy Source | Preferred Artifact ID |
| ----------------------- | ------------------------------ |
| `processed_transcript` | `narratio.transcript.polished` |
| `normalized_transcript` | `narratio.transcript.full` |
| `trimmed_transcript` | `narratio.transcript.trimmed` |
These aliases may be supported silently for v1.0. Documentation should prefer namespaced IDs.
### 9.3 Resolver Behavior
Artifact resolution should follow this order:
1. Normalize aliases to canonical artifact IDs.
2. Look for a current output reference in the session manifest.
3. Fall back to the canonical session-level path.
4. If the artifact is required, fail clearly if missing.
5. If the artifact is optional and missing, omit it from the downstream invocation.
6. Validate the artifact using the expected content validator.
7. Return a resolved artifact record containing ID, path, producer stage, output kind, and provenance.
Example conceptual result:
```json
{
"id": "narratio.transcript.trimmed",
"path": "/var/lib/narratio/work/dilfs/2026-05-07/transcripts/trimmed.json",
"producer_stage": "trim",
"producer_run_id": "20260517T174748Z-abcd1234",
"output_kind": "transcript_trimmed",
"content_type": "application/json"
}
```
### 9.4 Artifact Validation
Transcript artifacts must be valid JSON with a top-level `segments` array.
Markdown/text artifacts must exist and be non-empty when required.
Bounds artifacts must match the expected bounds schema and refer to segment IDs in the same transcript ID space used by the trim stage.
Validation should happen before a resolved artifact is passed to another stage or external subprocess.
## 10. Analyze Stage Implications
The analyze stage should consume artifacts through the artifact resolver.
Preferred Scriptorium config shape:
```yaml
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: "dnd.session_recap"
output_path: "artifacts/session_recap.md"
inputs:
transcript:
source: "narratio.transcript.trimmed"
required: true
```
Additional artifacts can choose different transcript tiers:
```yaml
scriptorium:
artifacts:
table_summary:
enabled: true
prompt_id: "dnd.table_summary"
output_path: "artifacts/table_summary.md"
inputs:
transcript:
source: "narratio.transcript.full"
required: true
```
For v1.0, Narratio does not need a generic DAG engine. It may execute configured analyze artifacts in deterministic order and allow later artifacts to consume earlier artifacts only when that relationship is explicit and unambiguous.
Rules:
* Artifact inputs resolve from current session-level durable state.
* Outputs are first written run-locally.
* Successful analyze outputs are promoted to session-level `artifacts/` paths.
* Manifest output refs record the producing run ID.
* Optional inputs are omitted when unavailable.
* Required missing inputs fail before invoking Scriptorium.
## 11. Archive Alignment
Local workspace semantics should mirror archive semantics.
Local session-level durable paths:
```text
work/{campaign}/{session}/transcripts/trimmed.json
work/{campaign}/{session}/artifacts/session_recap.md
work/{campaign}/{session}/current/manifest.json
work/{campaign}/{session}/current/run_id.txt
work/{campaign}/{session}/runs/{run_id}/...
```
should map naturally to remote archive paths:
```text
{root_prefix}/campaigns/{campaign}/sessions/{session}/transcripts/trimmed.json
{root_prefix}/campaigns/{campaign}/sessions/{session}/artifacts/session_recap.md
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/manifest.json
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/run_id.txt
{root_prefix}/campaigns/{campaign}/sessions/{session}/runs/{run_id}/...
```
The archive stage should publish run records and promoted current artifacts consistently with the local model.
`current/run_id.txt` remains the effective commit marker for the archived current session state.
## 12. Path Helper Requirements
All code should use centralized path helpers for workspace paths.
Stage code should not manually assemble durable cross-stage paths using raw string joins except through the path model.
Recommended helper surface:
```text
SessionRoot(campaignID, sessionID)
SessionManifestPath(campaignID, sessionID)
SessionCurrentDir(campaignID, sessionID)
SessionTranscriptsDir(campaignID, sessionID)
SessionArtifactsDir(campaignID, sessionID)
SessionReportsDir(campaignID, sessionID)
SessionLogsDir(campaignID, sessionID)
SessionConfigDir(campaignID, sessionID)
RunsDir(campaignID, sessionID)
RunRoot(campaignID, sessionID, runID)
RunManifestPath(campaignID, sessionID, runID)
RunStageDir(campaignID, sessionID, runID, stage)
RunStageOutputsDir(campaignID, sessionID, runID, stage)
RunStageLogsDir(campaignID, sessionID, runID, stage)
RunStageReportsDir(campaignID, sessionID, runID, stage)
RunStageConfigDir(campaignID, sessionID, runID, stage)
CanonicalArtifactPath(campaignID, sessionID, artifactID)
```
Path helpers should enforce safe relative paths for configured output paths:
* reject absolute paths unless explicitly allowed for a particular config field
* reject `..` traversal
* normalize separators
* preserve deterministic output paths
## 13. Directory Creation Policy
Directory creation should be centralized and idempotent.
Recommended policy:
* `prepare` ensures the baseline session directory structure exists.
* Every stage also calls shared layout helpers to ensure its required run-local directories exist before writing.
* `run-stage` should not depend on a prior `prepare` invocation merely to create folders.
* Missing directories should be created with appropriate permissions.
* Directory creation should not imply stage success.
This provides consistent layout while keeping direct stage execution robust.
## 14. Cleanup and Retention
Cleanup must preserve the distinction between durable session state and run history.
Workspace cleanup after successful archive may remove selected local directories only according to explicit configuration.
Potential retention policies:
```text
keep_all_runs
keep_failed_runs
keep_last_n_runs
delete_run_after_success
```
For v1.0, conservative retention is preferred:
* Do not delete durable session-level outputs unless explicitly requested.
* Do not delete failed run directories by default.
* If cleanup is enabled, remove only documented run-scoped or spool-scoped paths.
* Local development audio inputs must never be deleted by workspace cleanup.
## 15. Canonical-Only Layout Policy
Narratio now supports only the canonical campaign-aware layout:
```text
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/...
```
Legacy session-only layout compatibility is intentionally not implemented.
If legacy workspace data exists, operators should recreate or manually migrate that data outside Narratio before running v1.0 commands.
## 16. Documentation Updates Required
The following documentation should be updated to reflect this architecture:
* `README.md`
* `docs/architecture.md`
* a dedicated workspace/run-history document, such as this file
* S3/archive documentation
* analyze/artifact configuration documentation
* example pipeline files
Documentation should consistently use the following terms:
| Term | Meaning |
| ---------------- | ---------------------------------------------------------------------- |
| Session | Durable domain object and idempotency boundary. |
| Run | Execution attempt that may update session state. |
| Durable output | Canonical current session-level output. |
| Run-local output | Output produced inside a specific run directory before promotion. |
| Promotion | Validated copy/rename from run-local output to durable session output. |
| Session manifest | Current durable state of the session. |
| Run manifest | Execution record for a particular run. |
| Artifact ID | Symbolic source name resolved by the artifact registry. |
## 17. Architectural Invariants
The following invariants should hold after implementation:
1. `session_id` remains the idempotency boundary for normal operator commands.
2. `run_id` identifies an execution attempt, not the primary durable workspace.
3. Session-level canonical artifacts are the default inputs for downstream stages.
4. Run-local outputs are promoted only after validation.
5. A session's current durable state may be composed of outputs from multiple runs.
6. Sparse run directories are valid and expected.
7. The session manifest records current stage/artifact state and producer run IDs.
8. The run manifest records what happened during one invocation.
9. Artifact consumers resolve symbolic artifact IDs through a registry/resolver.
10. Local workspace semantics mirror S3 archive semantics.
11. Directory creation is centralized and idempotent.
12. Stage code uses path helpers rather than ad hoc path construction.
13. Forced upstream reruns invalidate downstream stage success unless downstream stages are rerun successfully.
14. Cleanup never removes durable session outputs or local development inputs unless explicitly configured to do so.
## 18. Implementation Guidance
A practical implementation sequence is:
1. Add this architecture document.
2. Add or revise path model helpers for session roots, run roots, stage directories, and canonical artifact paths.
3. Introduce session manifest versus run manifest concepts.
4. Route stage outputs through run-local directories.
5. Add promotion helpers with validation and atomic writes.
6. Update existing stages to promote durable outputs to session-level canonical paths.
7. Add artifact registry and resolver.
8. Update analyze to use artifact IDs and aliases.
9. Add simple downstream stale invalidation for forced upstream reruns.
10. Align archive/local path behavior and documentation.
11. Update examples and README.
12. Add tests for idempotency, sparse forced runs, promotion, manifest provenance, and artifact resolution.
This sequence intentionally avoids introducing a generic DAG engine. The v1.0 goal is a clear, deterministic, stage-oriented orchestrator with stable session-level outputs and inspectable run history.

View File

@@ -283,9 +283,9 @@ Session recap:
```bash
scriptorium run \
--prompt dnd.session_recap \
--input transcript=/work/session-42/transcript.polished.md \
--input glossary=/work/session-42/glossary.yml \
--out /work/session-42/artifacts/session_recap.md
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
--input glossary=/work/campaign-7/session-42/glossary.yml \
--out /work/campaign-7/session-42/artifacts/session_recap.md
```
Structured events:
@@ -293,8 +293,8 @@ Structured events:
```bash
scriptorium run \
--prompt dnd.structured_events \
--input transcript=/work/session-42/transcript.polished.md \
--out /work/session-42/artifacts/structured_events.json
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
--out /work/campaign-7/session-42/artifacts/structured_events.json
```
Glossary suggestions:
@@ -302,9 +302,9 @@ Glossary suggestions:
```bash
scriptorium run \
--prompt dnd.glossary_suggestions \
--input transcript=/work/session-42/transcript.polished.md \
--input previous_recap=/work/session-41/artifacts/session_recap.md \
--out /work/session-42/artifacts/glossary_suggestions.md
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
--input previous_recap=/work/campaign-7/session-41/artifacts/session_recap.md \
--out /work/campaign-7/session-42/artifacts/glossary_suggestions.md
```
Player-facing summary:
@@ -312,9 +312,9 @@ Player-facing summary:
```bash
scriptorium run \
--prompt dnd.player_summary \
--input transcript=/work/session-42/transcript.polished.md \
--input structured_events=/work/session-42/artifacts/structured_events.json \
--out /work/session-42/artifacts/player_summary.md
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
--input structured_events=/work/campaign-7/session-42/artifacts/structured_events.json \
--out /work/campaign-7/session-42/artifacts/player_summary.md
```
## 21. Non-Goals

File diff suppressed because it is too large Load Diff

119
docs/stages/archive.md Normal file
View File

@@ -0,0 +1,119 @@
# Archive Storage
This document describes implemented archive-stage publish behavior.
## S3 Paths
Session root:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
Run prefix:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
## Scope
Implemented:
- archive uploads successful run records to remote object storage through the storage backend abstraction.
- archive uploads configured promoted outputs to session-level keys.
- archive uploads `current/manifest.json`.
- archive uploads `current/run_id.txt` last as the effective commit marker.
- optional post-archive local cleanup:
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir
- tests use fake storage and do not require live S3.
Future work:
- `notify` stage behavior
- stale detection
- optional future source-audio upload mode
- additional artifact generation beyond current implemented set
## Prerequisites
Archive verifies these stages succeeded before upload:
- `prepare`
- `transcribe`
- `merge`
- `polish`
- `normalize`
- `trim`
- `analyze`
If any prerequisite is missing or not succeeded, archive fails and does not upload.
Failed or incomplete runs remain local only.
## Run Upload
Archive uploads existing files from the run workdir when present:
- `inputs/`
- `transcripts/`
- `artifacts/`
- `reports/` (optional)
- `config/`
- `logs/`
- `manifest.json`
Relative paths are preserved under `runs/{run_id}/`.
## Promotion Rules
Archive applies `archive.promote_artifacts` in config order.
Rule behavior:
- `from`: local workdir-relative source path
- `to`: session-root-relative destination key
- `required: true`: missing source fails archive
- `required: false`: missing source is skipped and recorded
Default promoted outputs:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
## Current Pointers
Archive writes:
1. `current/manifest.json` (after run upload + promotions)
2. `current/run_id.txt` last
`current/run_id.txt` contains exactly:
- `{run_id}` plus trailing newline
Writing `current/run_id.txt` last makes it the effective commit marker for published session state.
If any required run upload, promotion upload, or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`.
Cleanup runs only after this commit-marker write has succeeded.
## Audio Upload Policy
Archive does not upload local `audio/` by default.
Original audio is expected at the session-level audio prefix and is not duplicated under `runs/{run_id}/`.
## Config Controls
- `archive.enabled: false` skips archive cleanly.
- `archive.upload_run: false` skips run upload cleanly.
- both skip cases also skip post-archive local cleanup.
## Metadata
Archive stage metadata includes non-secret upload context (for example):
- `s3_bucket`
- `s3_run_prefix`
- run upload counts/paths
- promoted upload counts/paths
- skipped optional promotions
- `current_manifest_key`
- `current_run_id_key`
- `current_pointer_written`
- `audio_upload_skipped`

84
docs/stages/prepare.md Normal file
View File

@@ -0,0 +1,84 @@
# S3 Audio Input
This document describes implemented S3 audio input behavior in `prepare`.
## Scope
Implemented:
- `prepare` can acquire source audio from S3 when `session.inputs.audio_s3.prefix` is configured.
- object listing and download go through the storage backend abstraction.
- tests use fake storage; no live S3 service is required for test runs.
Not implemented:
- uploads of failed runs
## Required Configuration
`pipeline.yml`:
- `storage.s3.bucket` must be set when S3 audio input is used.
- `storage.s3.root_prefix` defaults to `dnd`.
- `storage.s3.access_key_id_env` defaults to `OBJECT_STORAGE_KEY_ID`.
- `storage.s3.secret_access_key_env` defaults to `OBJECT_STORAGE_KEY`.
- `spool.root` defaults to `/var/spool/narratio`.
`session.yml`:
- configure `session.campaign` and `session.session_id`.
- configure `session.inputs.audio_s3.prefix` for S3 audio input.
- do not configure `inputs.audio_dir` or `inputs.audio_files` at the same time as `inputs.audio_s3`.
## Prefix Shape
Session S3 root:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
Audio prefix:
`{session_root}/{audio_s3.prefix}`
Example:
`dnd/campaigns/forsaken/sessions/2026-04-19/audio/`
Audio files must already exist in S3 before running Narratio.
## Prepare Behavior
When `inputs.audio_s3.prefix` is configured, `prepare`:
1. lists objects under the computed S3 audio prefix
2. filters to `.flac` objects
3. fails when no `.flac` objects are found
4. downloads selected objects to spool audio:
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
5. materializes audio into workdir audio:
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/audio/`
6. records input provenance in the manifest (bucket, key, metadata, local paths, checksum)
Notes:
- `.flac` filtering is case-insensitive.
- ETag is recorded as provider metadata only and is not treated as a checksum.
## Local Audio Development
Local audio workflows remain supported:
- `inputs.audio_dir`
- `inputs.audio_files`
These options are mutually exclusive with `inputs.audio_s3`.
## Archive Boundary
Current archive behavior relevant to S3 audio input:
- successful runs are uploaded by archive under `runs/{run_id}/`
- configured promotions are uploaded to session-level destinations
- `current/manifest.json` and `current/run_id.txt` are published
- local source audio is not re-uploaded by default
- failed or incomplete runs are not uploaded

View File

@@ -0,0 +1,26 @@
workspace:
root: ./tmp/narratio-workspace
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
seriatim:
binary: "seriatim"
timeout: "10m"
output_schema: "seriatim-intermediate"
coalesce_gap: 3.0
audita:
binary: "audita"
timeout: "3h"
base_url: "https://openrouter.ai/api/v1"
model: "openrouter/google/gemma-4-31b-it"
llm_api_key_env: "AUDITA_LLM_API_KEY"
modules: ["glossary", "homophones", "spoken_word", "grammar"]
output_schema: "audita-v1"
work_dir_retention: "auto"
total_llm_concurrency: 2
proposal_llm_concurrency: 1
validation_model: "openrouter/google/gemma-4-31b-it"
validation_llm_concurrency: 1
report: true

View File

@@ -1,102 +1,51 @@
workspace:
root: ./tmp/narratio-workspace
cleanup_after_archive: false
storage:
backend: local
backend: s3
s3:
bucket: "my-dnd-archive"
root_prefix: "dnd"
region: "us-east-1"
# Optional credential env-var names (defaulted when omitted):
# access_key_id_env: "OBJECT_STORAGE_KEY_ID"
# secret_access_key_env: "OBJECT_STORAGE_KEY"
secrets:
# Optional: load environment variables from files in this directory.
# File name = env var name; file contents = env var value.
env_dir: /var/local/narratio/secrets
spool:
root: "/var/spool/narratio"
delete_audio_after_archive: false
archive:
enabled: true
upload_run: true
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
language: "en"
timeout: "30m"
retries: 3
retry_delay: "2s"
concurrency: 2
seriatim:
binary: "seriatim"
timeout: "10m"
output_schema: "seriatim-intermediate"
coalesce_gap: 3.0
report: true
env:
overlap_word_run_gap: 1.0
overlap_word_run_reorder_window: 1.0
backchannel_max_duration: 2.0
filler_max_duration: 1.25
# Optional. When omitted entirely, Narratio defaults to seriatim binary + runtime defaults.
seriatim: {}
# Optional runtime overrides. Model/provider can be owned by Audita runtime config.
audita:
binary: "audita"
timeout: "3h"
config_path: "/usr/local/etc/audita/config.yml"
llm_api_key_env: "AUDITA_LLM_API_KEY"
modules:
- glossary
- homophones
- glossary
- spoken_word
- grammar
- homophones
- glossary
base_url: "https://openrouter.ai/api/v1"
model: "openrouter/google/gemma-4-31b-it"
llm_concurrency: 1
validation_model: ""
validation_llm_concurrency: 1
report: true
normalize:
# Session-workdir-relative when not absolute.
output_path: "transcripts/normalized.json"
output_schema: "seriatim-intermediate"
report: true
trim:
enabled: true
# Session-workdir-relative when not absolute.
output_path: "transcripts/trimmed.json"
bounds:
prompt_id: "dnd_session.bounds"
# Empty means use prompt default profile.
profile_id: ""
transcript_input_name: "transcript"
output_path: "artifacts/session_bounds.json"
timeout: "10m"
render_debug: false
render_output_path: "artifacts/session_bounds.render.json"
seriatim:
report: false
# Optional Scriptorium integration for analyze artifacts.
scriptorium:
binary: "scriptorium"
config_path: "/etc/scriptorium/config.yml"
timeout: "10m"
render_debug: false
config_path: "/usr/local/etc/scriptorium/config.yml"
artifacts:
session_recap:
enabled: true
prompt_id: "dnd.session_recap"
profile_id: "local-quality"
output_path: "artifacts/session_recap.md"
timeout: "10m"
# Optional per-artifact override of global scriptorium.render_debug.
# render_debug: true
inputs:
transcript:
# Available transcript sources:
# - trimmed_transcript (recommended for session_recap)
# - normalized_transcript (recommended for future full-session analysis)
# - processed_transcript (raw Audita-polished output)
source: "trimmed_transcript"
required: true
previous_recap:
source: "previous_session_artifact"
artifact: "session_recap"
# Optional: set when previous recap is available.
path: ""
required: false
vars:
session_id: true
@@ -104,11 +53,3 @@ scriptorium:
campaign_name: true
previous_session_id: true
output_kind: "session_recap"
analyzer:
timeout: 20m
artifacts:
output_dir: artifacts
notification:
timeout: 10s

View File

@@ -4,7 +4,10 @@ date: 2026-05-03
title: Sample Session
inputs:
audio_dir: ./audio
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
# Narratio prepare lists this prefix and downloads .flac files.
# audio_s3:
# prefix: "audio/"
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

View File

@@ -0,0 +1,12 @@
session_id: "{{ session_id }}"
campaign: sample-campaign
date: ""
title: ""
inputs:
audio_dir: ./audio
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
# audio_s3:
# prefix: "audio/{{ session_id }}/"
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

25
go.mod
View File

@@ -2,4 +2,27 @@ module gitea.maximumdirect.net/eric/narratio
go 1.25.0
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/smithy-go v1.25.1
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
)

36
go.sum
View File

@@ -1,3 +1,39 @@
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -24,6 +24,12 @@ type PolishRequest struct {
Modules []string
BaseURL string
Model string
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
StdoutLogPath string

View File

@@ -21,7 +21,12 @@ type SubprocessRunnerConfig struct {
Modules []string
BaseURL string
Model string
LLMConcurrency *int
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
Report bool
@@ -35,7 +40,12 @@ type SubprocessRunner struct {
modules []string
baseURL string
model string
llmConcurrency *int
transcriptDescription string
configPath string
outputSchema string
workDirRetention string
totalLLMConcurrency *int
proposalLLMConcurrency *int
validationModel string
validationLLMConcurrency *int
report bool
@@ -49,7 +59,12 @@ func NewSubprocessRunnerFromConfigValues(
modules []string,
baseURL string,
model string,
llmConcurrency *int,
transcriptDescription string,
configPath string,
outputSchema string,
workDirRetention string,
totalLLMConcurrency *int,
proposalLLMConcurrency *int,
validationModel string,
validationLLMConcurrency *int,
report bool,
@@ -68,7 +83,12 @@ func NewSubprocessRunnerFromConfigValues(
Modules: modules,
BaseURL: baseURL,
Model: model,
LLMConcurrency: llmConcurrency,
TranscriptDescription: transcriptDescription,
ConfigPath: configPath,
OutputSchema: outputSchema,
WorkDirRetention: workDirRetention,
TotalLLMConcurrency: totalLLMConcurrency,
ProposalLLMConcurrency: proposalLLMConcurrency,
ValidationModel: validationModel,
ValidationLLMConcurrency: validationLLMConcurrency,
Report: report,
@@ -83,33 +103,39 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
if cfg.Timeout <= 0 {
return nil, fmt.Errorf("audita timeout must be > 0")
}
if len(cfg.Modules) == 0 {
return nil, fmt.Errorf("audita modules must include at least one module")
}
for i, module := range cfg.Modules {
if strings.TrimSpace(module) == "" {
return nil, fmt.Errorf("audita module at index %d is empty", i)
}
}
if strings.TrimSpace(cfg.BaseURL) == "" {
return nil, fmt.Errorf("audita base url is required")
}
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil {
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
if strings.TrimSpace(cfg.BaseURL) != "" {
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil {
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
}
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
}
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
}
if strings.TrimSpace(cfg.Model) == "" {
return nil, fmt.Errorf("audita model is required")
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
}
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
}
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
}
switch strings.TrimSpace(cfg.OutputSchema) {
case "", "bare-segments", "audita-v1":
default:
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
}
switch strings.TrimSpace(cfg.WorkDirRetention) {
case "", "always", "auto", "never":
default:
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
}
modules := make([]string, len(cfg.Modules))
for i, m := range cfg.Modules {
@@ -123,7 +149,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
modules: modules,
baseURL: strings.TrimSpace(cfg.BaseURL),
model: strings.TrimSpace(cfg.Model),
llmConcurrency: cfg.LLMConcurrency,
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
configPath: strings.TrimSpace(cfg.ConfigPath),
outputSchema: strings.TrimSpace(cfg.OutputSchema),
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
totalLLMConcurrency: cfg.TotalLLMConcurrency,
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
validationModel: strings.TrimSpace(cfg.ValidationModel),
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
report: cfg.Report,
@@ -152,7 +183,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
}
reqModules := req.Modules
if len(reqModules) == 0 {
if reqModules == nil {
reqModules = append([]string(nil), r.modules...)
}
args := r.buildArgs(req, reqModules)
@@ -168,14 +199,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
env["AUDITA_LLM_API_KEY"] = credential
credentialPresent = true
}
primaryConcurrencyViaEnv := false
if r.llmConcurrency != nil {
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
primaryConcurrencyViaEnv = true
}
if req.GeneratedConfigPath != "" {
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
}
}
@@ -196,7 +222,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
req.StderrLogPath,
)
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
"%s: %w",
wrappedMessage,
err,
@@ -204,11 +230,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
}
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
}
if r.report {
if err := validateJSONFile(req.ReportPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
}
}
@@ -223,21 +249,25 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
Duration: runRes.Duration,
InvokedBinary: r.binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": reqModules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
"adapter": "audita_subprocess",
"modules": reqModules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
},
}, nil
}
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
return PolishResult{
ProcessedTranscriptPath: req.OutputProcessedPath,
ReportPath: req.ReportPath,
@@ -249,16 +279,20 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
Duration: runRes.Duration,
InvokedBinary: r.binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
"adapter": "audita_subprocess",
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
},
}
}
@@ -269,14 +303,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--modules", strings.Join(modules, ","),
"--base-url", r.baseURL,
"--model", r.model,
"--work-dir", req.WorkDir,
}
if r.baseURL != "" {
args = append(args, "--base-url", r.baseURL)
}
if r.model != "" {
args = append(args, "--model", r.model)
}
if len(modules) > 0 {
args = append(args, "--modules", strings.Join(modules, ","))
}
if r.report {
args = append(args, "--report-json", req.ReportPath)
}
if r.transcriptDescription != "" {
args = append(args, "--transcript-description", r.transcriptDescription)
}
if r.configPath != "" {
args = append(args, "--config", r.configPath)
}
if r.outputSchema != "" {
args = append(args, "--output-schema", r.outputSchema)
}
if r.workDirRetention != "" {
args = append(args, "--work-dir-retention", r.workDirRetention)
}
if r.totalLLMConcurrency != nil {
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
}
if r.proposalLLMConcurrency != nil {
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
}
if r.validationModel != "" {
args = append(args, "--validation-model", r.validationModel)
}
@@ -286,29 +344,31 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
return args
}
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
payload := map[string]any{
"schema": "audita.generated.v1",
"binary": r.binary,
"args": args,
"timeout": r.timeout.String(),
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"report_enabled": r.report,
"merged_transcript_path": req.MergedTranscriptPath,
"glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath,
"report_path": req.ReportPath,
"work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
}
if r.llmConcurrency != nil {
payload["llm_concurrency"] = *r.llmConcurrency
"schema": "audita.generated.v1",
"binary": r.binary,
"args": args,
"timeout": r.timeout.String(),
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"report_enabled": r.report,
"merged_transcript_path": req.MergedTranscriptPath,
"glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath,
"report_path": req.ReportPath,
"work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
}

View File

@@ -25,7 +25,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
wrapper := writeAuditaHelperWrapper(t)
llmConcurrency := 1
totalLLMConcurrency := 3
proposalLLMConcurrency := 2
validationLLMConcurrency := 2
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
Binary: wrapper,
@@ -34,7 +35,12 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
Modules: []string{"glossary", "homophones", "glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
TranscriptDescription: "Campaign Session 42",
ConfigPath: "/etc/audita/config.yml",
OutputSchema: "audita-v1",
WorkDirRetention: "auto",
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
ValidationModel: "openrouter/google/gemma-4-31b-it",
ValidationLLMConcurrency: &validationLLMConcurrency,
Report: true,
@@ -93,11 +99,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
"process", req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--modules", "glossary,homophones,glossary",
"--work-dir", req.WorkDir,
"--base-url", "https://openrouter.ai/api/v1",
"--model", "openrouter/google/gemma-4-31b-it",
"--work-dir", req.WorkDir,
"--modules", "glossary,homophones,glossary",
"--report-json", req.ReportPath,
"--transcript-description", "Campaign Session 42",
"--config", "/etc/audita/config.yml",
"--output-schema", "audita-v1",
"--work-dir-retention", "auto",
"--total-llm-concurrency", "3",
"--proposal-llm-concurrency", "2",
"--validation-model", "openrouter/google/gemma-4-31b-it",
"--validation-llm-concurrency", "2",
}
@@ -107,8 +119,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
}
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
}
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
@@ -124,16 +136,14 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
@@ -155,16 +165,14 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
@@ -211,6 +219,65 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
}
}
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--modules" {
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--base-url" {
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
}
if rec.Args[i] == "--model" {
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
@@ -220,16 +287,14 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -256,16 +321,14 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -286,16 +349,14 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -316,16 +377,14 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -346,16 +405,14 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -376,16 +433,14 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -398,11 +453,11 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
}
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected binary validation error")
}
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected timeout parse error")
}

View File

@@ -0,0 +1,29 @@
package storage
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// NewObjectStoreFromConfig constructs a remote object store from resolved config.
func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) {
if cfg == nil || cfg.Pipeline == nil {
return nil, fmt.Errorf("pipeline config is required")
}
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
if cfg.Pipeline.Storage.S3 == nil {
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
}
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
return nil, fmt.Errorf("no remote object store backend is configured")
}

View File

@@ -0,0 +1,56 @@
package storage
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
return &fakeS3API{}, nil
}
store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{
Backend: "s3",
S3: &config.StorageS3Config{
Bucket: "my-archive",
},
},
},
})
if err != nil {
t.Fatalf("NewObjectStoreFromConfig() error = %v", err)
}
if _, ok := store.(*S3Backend); !ok {
t.Fatalf("store type = %T, want *S3Backend", store)
}
}
func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{Backend: "s3"},
},
})
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err)
}
}
func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{Backend: "local"},
},
})
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
}
}

View File

@@ -1,6 +1,14 @@
package storage
import "context"
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// NoopBackend is a deterministic no-op archive/storage adapter.
type NoopBackend struct{}
@@ -18,6 +26,21 @@ type FakeBackend struct {
Requests []ArchiveRequest
Err error
Result ArchiveResult
Objects map[string]FakeObject
Uploads []FakeUploadCall
ListErr error
DownloadErr error
UploadErr error
ExistsErr error
}
// FakeUploadCall captures one upload invocation in call order.
type FakeUploadCall struct {
LocalPath string
Key string
Options UploadOptions
}
// Archive records request and returns configured response.
@@ -38,3 +61,148 @@ func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveR
}
return res, nil
}
// FakeObject is a deterministic fake object-store record.
type FakeObject struct {
Key string
Data []byte
Metadata map[string]string
ETag string
LastModified *time.Time
}
// SeedObject inserts or replaces an object in the fake object store.
func (f *FakeBackend) SeedObject(obj FakeObject) {
if f.Objects == nil {
f.Objects = map[string]FakeObject{}
}
key := normalizeObjectKey(obj.Key)
obj.Key = key
obj.Data = append([]byte(nil), obj.Data...)
obj.Metadata = copyMetadata(obj.Metadata)
f.Objects[key] = obj
}
// List returns deterministic prefix-filtered objects.
func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if f.ListErr != nil {
return nil, f.ListErr
}
normalizedPrefix := normalizeObjectKey(prefix)
keys := make([]string, 0, len(f.Objects))
for key := range f.Objects {
if strings.HasPrefix(key, normalizedPrefix) {
keys = append(keys, key)
}
}
sort.Strings(keys)
out := make([]ObjectInfo, 0, len(keys))
for _, key := range keys {
obj := f.Objects[key]
out = append(out, ObjectInfo{
Key: obj.Key,
Size: int64(len(obj.Data)),
ETag: obj.ETag,
LastModified: obj.LastModified,
})
}
return out, nil
}
// Download writes one object to a local path.
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
if err := ctx.Err(); err != nil {
return err
}
if f.DownloadErr != nil {
return f.DownloadErr
}
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
obj, ok := f.Objects[normalizeObjectKey(key)]
if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
}
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
}
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
return fmt.Errorf("download object %q: write local file: %w", key, err)
}
return nil
}
// Upload reads a local file and stores it under key.
func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, err
}
if f.UploadErr != nil {
return ObjectInfo{}, f.UploadErr
}
if strings.TrimSpace(localPath) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
}
if strings.TrimSpace(key) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := os.ReadFile(localPath)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
}
normalizedKey := normalizeObjectKey(key)
f.Uploads = append(f.Uploads, FakeUploadCall{
LocalPath: localPath,
Key: normalizedKey,
Options: UploadOptions{
Metadata: copyMetadata(opts.Metadata),
ContentType: opts.ContentType,
},
})
now := time.Now().UTC()
obj := FakeObject{
Key: normalizedKey,
Data: data,
Metadata: copyMetadata(opts.Metadata),
LastModified: &now,
}
f.SeedObject(obj)
return ObjectInfo{
Key: normalizedKey,
Size: int64(len(data)),
LastModified: &now,
}, nil
}
// Exists checks object presence.
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
if f.ExistsErr != nil {
return false, f.ExistsErr
}
_, ok := f.Objects[normalizeObjectKey(key)]
return ok, nil
}
func copyMetadata(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for k, v := range in {
out[k] = v
}
return out
}

View File

@@ -3,6 +3,9 @@ package storage
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -29,3 +32,84 @@ func TestFakeBackendError(t *testing.T) {
t.Fatal("expected error, got nil")
}
}
func TestFakeBackendListPrefixFiltering(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")})
fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")})
items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/")
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 2 {
t.Fatalf("List() len = %d, want 2", len(items))
}
if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" {
t.Fatalf("List() keys = %#v", items)
}
}
func TestFakeBackendDownload(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
t.Fatalf("Download() error = %v", err)
}
data, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "audio-a" {
t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a")
}
}
func TestFakeBackendUploadAndExists(t *testing.T) {
fake := &FakeBackend{}
local := filepath.Join(t.TempDir(), "upload.txt")
if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
Metadata: map[string]string{"kind": "artifact"},
})
if err != nil {
t.Fatalf("Upload() error = %v", err)
}
if info.Key != "runs/id/artifact.txt" {
t.Fatalf("Upload() key = %q, want normalized key", info.Key)
}
ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if !ok {
t.Fatal("Exists() = false, want true")
}
}
func TestFakeBackendObjectErrors(t *testing.T) {
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
t.Fatalf("List() error = %v, want list fail", err)
}
if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") {
t.Fatalf("Download() error = %v, want download fail", err)
}
local := filepath.Join(t.TempDir(), "x.txt")
_ = os.WriteFile(local, []byte("x"), 0o644)
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
t.Fatalf("Upload() error = %v, want upload fail", err)
}
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
t.Fatalf("Exists() error = %v, want exists fail", err)
}
}

View File

@@ -0,0 +1,8 @@
package storage
import "strings"
func normalizeObjectKey(key string) string {
normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/")
return strings.TrimLeft(normalized, "/")
}

View File

@@ -0,0 +1,21 @@
package storage
import "testing"
func TestNormalizeObjectKey(t *testing.T) {
tests := []struct {
in string
want string
}{
{in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"},
{in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"},
{in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"},
{in: "", want: ""},
}
for _, tt := range tests {
if got := normalizeObjectKey(tt.in); got != tt.want {
t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}

View File

@@ -0,0 +1,32 @@
package storage
import (
"context"
"time"
)
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
//
// Key invariant:
// callers pass full bucket-relative object keys. Backend implementations do not
// infer Narratio session semantics and do not prepend root prefixes.
type ObjectStore interface {
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
Download(ctx context.Context, key, localPath string) error
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
Exists(ctx context.Context, key string) (bool, error)
}
// ObjectInfo describes one object in remote storage.
type ObjectInfo struct {
Key string
Size int64
ETag string
LastModified *time.Time
}
// UploadOptions configures optional object upload metadata.
type UploadOptions struct {
Metadata map[string]string
ContentType string
}

View File

@@ -0,0 +1,275 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type s3API interface {
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
}
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
type S3Backend struct {
bucket string
client s3API
}
type s3ClientOptions struct {
Region string
Endpoint string
ForcePathStyle bool
AccessKeyID string
SecretKey string
}
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
if strings.TrimSpace(opts.Region) != "" {
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
}
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
strings.TrimSpace(opts.AccessKeyID),
strings.TrimSpace(opts.SecretKey),
"",
),
))
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
if err != nil {
return nil, fmt.Errorf("load aws config: %w", err)
}
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if strings.TrimSpace(opts.Endpoint) != "" {
endpoint := strings.TrimSpace(opts.Endpoint)
o.BaseEndpoint = &endpoint
}
o.UsePathStyle = opts.ForcePathStyle
}), nil
}
// NewS3BackendFromConfig builds an S3 backend from resolved config.
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
bucket := strings.TrimSpace(cfg.Bucket)
if bucket == "" {
return nil, fmt.Errorf("storage.s3.bucket is required")
}
client, err := newS3Client(ctx, s3ClientOptions{
Region: cfg.Region,
Endpoint: cfg.Endpoint,
ForcePathStyle: cfg.ForcePathStyle,
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
})
if err != nil {
return nil, fmt.Errorf("build s3 client: %w", err)
}
return &S3Backend{
bucket: bucket,
client: client,
}, nil
}
func s3CredentialFromEnv(envVarName string) string {
name := strings.TrimSpace(envVarName)
if name == "" {
return ""
}
value, ok := os.LookupEnv(name)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func orDefaultEnvName(name, fallback string) string {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
return fallback
}
return trimmed
}
// List returns objects under prefix.
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
normalizedPrefix := normalizeObjectKey(prefix)
out := make([]ObjectInfo, 0)
var token *string
for {
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: &b.bucket,
Prefix: &normalizedPrefix,
ContinuationToken: token,
})
if err != nil {
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
}
for _, item := range resp.Contents {
var lastModified *time.Time
if item.LastModified != nil {
t := *item.LastModified
lastModified = &t
}
out = append(out, ObjectInfo{
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
Size: valueOrZeroInt64(item.Size),
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
LastModified: lastModified,
})
}
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
break
}
token = resp.NextContinuationToken
}
return out, nil
}
// Download retrieves one object to localPath, creating parent directories as needed.
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
normalizedKey := normalizeObjectKey(key)
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
if err != nil {
return fmt.Errorf("download object %q: %w", normalizedKey, err)
}
defer resp.Body.Close()
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
}
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
}
defer dst.Close()
if _, err := io.Copy(dst, resp.Body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
}
if err := dst.Sync(); err != nil {
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
}
return nil
}
// Upload sends a local file to key.
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
normalizedKey := normalizeObjectKey(key)
if strings.TrimSpace(localPath) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
}
if normalizedKey == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
file, err := os.Open(localPath)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
}
input := &s3.PutObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
Body: file,
Metadata: copyMetadata(opts.Metadata),
}
if strings.TrimSpace(opts.ContentType) != "" {
ct := strings.TrimSpace(opts.ContentType)
input.ContentType = &ct
}
resp, err := b.client.PutObject(ctx, input)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
}
return ObjectInfo{
Key: normalizedKey,
Size: stat.Size(),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
}, nil
}
// Exists checks whether one object key exists.
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
normalizedKey := normalizeObjectKey(key)
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
if err == nil {
return true, nil
}
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NotFound", "NoSuchKey", "404":
return false, nil
}
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
}
func valueOrEmpty(v *string) string {
if v == nil {
return ""
}
return *v
}
func valueOrZeroInt64(v *int64) int64 {
if v == nil {
return 0
}
return *v
}
func valueOrFalseBool(v *bool) bool {
if v == nil {
return false
}
return *v
}

View File

@@ -0,0 +1,253 @@
package storage
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type fakeS3API struct {
listOut *s3.ListObjectsV2Output
listErr error
getBody io.ReadCloser
getErr error
putOut *s3.PutObjectOutput
putErr error
headErr error
lastList *s3.ListObjectsV2Input
lastGet *s3.GetObjectInput
lastPut *s3.PutObjectInput
lastHead *s3.HeadObjectInput
}
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
f.lastList = params
if f.listErr != nil {
return nil, f.listErr
}
if f.listOut == nil {
return &s3.ListObjectsV2Output{}, nil
}
return f.listOut, nil
}
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
f.lastGet = params
if f.getErr != nil {
return nil, f.getErr
}
body := f.getBody
if body == nil {
body = io.NopCloser(strings.NewReader(""))
}
return &s3.GetObjectOutput{Body: body}, nil
}
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
f.lastPut = params
if f.putErr != nil {
return nil, f.putErr
}
if f.putOut == nil {
return &s3.PutObjectOutput{}, nil
}
return f.putOut, nil
}
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
f.lastHead = params
if f.headErr != nil {
return nil, f.headErr
}
return &s3.HeadObjectOutput{}, nil
}
func TestS3BackendListAndKeyNormalization(t *testing.T) {
lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC)
client := &fakeS3API{
listOut: &s3.ListObjectsV2Output{
Contents: []types.Object{
{Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified},
},
},
}
backend := &S3Backend{bucket: "bucket-1", client: client}
items, err := backend.List(context.Background(), `dnd\campaigns\`)
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 1 {
t.Fatalf("List() len = %d, want 1", len(items))
}
if items[0].Key != "dnd/campaigns/forsaken/a.flac" {
t.Fatalf("List() key = %q, want normalized slash key", items[0].Key)
}
if items[0].ETag != "abc" {
t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc")
}
if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" {
t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList)
}
}
func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))}
backend := &S3Backend{bucket: "bucket-1", client: client}
dst := filepath.Join(t.TempDir(), "nested", "clip.flac")
if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil {
t.Fatalf("Download() error = %v", err)
}
data, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "audio" {
t.Fatalf("downloaded content = %q, want %q", string(data), "audio")
}
if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" {
t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet)
}
}
func TestS3BackendUploadAndExists(t *testing.T) {
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
backend := &S3Backend{bucket: "bucket-1", client: client}
local := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
Metadata: map[string]string{"kind": "artifact"},
})
if err != nil {
t.Fatalf("Upload() error = %v", err)
}
if info.Key != "runs/id/artifact.txt" {
t.Fatalf("Upload key = %q, want normalized key", info.Key)
}
if info.ETag != "etag123" {
t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123")
}
if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" {
t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut)
}
ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if !ok {
t.Fatal("Exists() = false, want true")
}
}
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
if err == nil || !strings.Contains(err.Error(), "no such file") {
t.Fatalf("Upload() error = %v, want missing local file error", err)
}
}
func TestS3BackendExistsNotFound(t *testing.T) {
backend := &S3Backend{
bucket: "bucket-1",
client: &fakeS3API{
headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"},
},
}
ok, err := backend.Exists(context.Background(), "missing-key")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if ok {
t.Fatal("Exists() = true, want false")
}
}
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
var got s3ClientOptions
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
got = opts
return &fakeS3API{}, nil
}
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
Endpoint: "http://localhost:9000",
ForcePathStyle: true,
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
}
if backend.bucket != "my-archive" {
t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive")
}
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
}
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
}
}
func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{})
if err == nil || !strings.Contains(err.Error(), "bucket is required") {
t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err)
}
}
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
var got s3ClientOptions
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
got = opts
return &fakeS3API{}, nil
}
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
SecretKeyEnv: "MISSING_SECRET_KEY",
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
}
if got.AccessKeyID != "" || got.SecretKey != "" {
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
}
}
func strPtr(v string) *string { return &v }
func int64Ptr(v int64) *int64 { return &v }
var _ s3API = (*fakeS3API)(nil)

View File

@@ -64,12 +64,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
args []string
want string
}{
{name: "run missing flags", args: []string{"run"}, want: "run: --session is required"},
{name: "plan missing flags", args: []string{"plan"}, want: "plan: --session is required"},
{name: "run missing flags", args: []string{"run"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "plan missing flags", args: []string{"plan"}, want: "plan: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
{name: "resume missing flags", args: []string{"resume"}, want: "resume: --session is required"},
{name: "resume missing flags", args: []string{"resume"}, want: "resume: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: --session is required"},
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "run missing config uses defaults", args: []string{"run", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
}
@@ -111,7 +111,7 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
var stdout bytes.Buffer
@@ -156,7 +156,7 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
t.Fatal("expected whisperx server to be called at least once")
}
outPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "transcripts", "raw", "alice.json")
outPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "transcripts", "raw", "alice.json")
data, err := os.ReadFile(outPath)
if err != nil {
t.Fatalf("ReadFile(%q): %v", outPath, err)
@@ -208,6 +208,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: ` + sessionID + `
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
@@ -232,7 +233,7 @@ inputs:
_ = os.Chdir(originalWD)
})
workRoot := filepath.Join(workspaceRoot, "work", sessionID)
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
@@ -271,6 +272,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
@@ -377,6 +379,11 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
root: ` + workspaceRoot + `
storage:
backend: s3
s3:
bucket: test-bucket
archive:
enabled: true
upload_run: false
whisperx:
transcribe_url: ` + url + `
timeout: 2s
@@ -400,6 +407,7 @@ notification:
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml

View File

@@ -21,9 +21,11 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -32,16 +34,18 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("plan: unexpected positional arguments")
}
if sessionPath == "" {
return fmt.Errorf("plan: --session is required")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
cfg, err := config.Load(resolvedPipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("plan: %w", err)
}
@@ -53,7 +57,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
}
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
paths, err := store.EnsureLayout(cfg.Session.SessionID)
paths, err := store.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if err != nil {
return fmt.Errorf("plan: prepare workdir: %w", err)
}

View File

@@ -36,7 +36,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
t.Fatalf("first output = %q, want totals", got)
}
sessionWorkdir := artifacts.SessionWorkDir(workspaceRoot, "2026-05-03")
sessionWorkdir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
expectedDirs := []string{
sessionWorkdir,
filepath.Join(sessionWorkdir, "inputs"),
@@ -63,7 +63,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
@@ -113,6 +113,7 @@ notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml

View File

@@ -0,0 +1,208 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
return nil
}
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterArchive
workRequested := env.Config.Pipeline.Workspace.CleanupAfterArchive
if !spoolRequested && !workRequested {
return nil
}
sr := archiveStageRecordForCleanup(m, executed)
if sr == nil {
return nil
}
if sr.Metadata == nil {
sr.Metadata = map[string]any{}
}
sr.Metadata["spool_cleanup_requested"] = spoolRequested
sr.Metadata["workdir_cleanup_requested"] = workRequested
eligible, reason := archiveCleanupEligible(env.Config, sr)
if !eligible {
sr.Metadata["cleanup_skipped"] = true
sr.Metadata["cleanup_skipped_reason"] = reason
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
}
return nil
}
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolDir == "" {
spoolDir = artifacts.SessionSpoolAudioDir(
env.Config.Pipeline.Spool.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
)
}
workDir := strings.TrimSpace(m.LocalWorkDir)
if workDir == "" {
workDir = artifacts.SessionRunRootForCampaign(
env.Config.Pipeline.Workspace.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
)
}
if spoolRequested {
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_archive"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_archive"
sr.Metadata["cleanup_failed_path"] = spoolDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
}
if !workRequested {
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
}
return nil
}
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_archive"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_archive"
sr.Metadata["cleanup_failed_path"] = workDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
return nil
}
func archiveStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
if m == nil {
return nil
}
archiveRan := false
for _, name := range executed {
if name == "archive" {
archiveRan = true
break
}
}
if !archiveRan {
return nil
}
sr := m.Stages["archive"]
if sr == nil || sr.Status != manifest.StatusSucceeded {
return nil
}
return sr
}
func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
return false, "archive configuration is missing"
}
enabled := true
if cfg.Pipeline.Archive.Enabled != nil {
enabled = *cfg.Pipeline.Archive.Enabled
}
if !enabled {
return false, "archive.enabled is false"
}
uploadRun := true
if cfg.Pipeline.Archive.UploadRun != nil {
uploadRun = *cfg.Pipeline.Archive.UploadRun
}
if !uploadRun {
return false, "archive.upload_run is false"
}
if sr == nil || sr.Metadata == nil {
return false, "archive metadata is missing"
}
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
return false, "archive stage was skipped"
}
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
return false, "archive did not upload run record"
}
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
return false, "archive did not write current pointer"
}
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
return false, "archive current run pointer key is missing"
}
return true, ""
}
func removeRunScopedDir(root, target, policy string) error {
cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" {
return fmt.Errorf("cleanup policy %s: root path is required", policy)
}
if cleanTarget == "" {
return fmt.Errorf("cleanup policy %s: target path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
targetAbs, err := filepath.Abs(cleanTarget)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
}
rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil {
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
}
if rel == "." {
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
}
info, err := os.Lstat(targetAbs)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
}
if !info.IsDir() {
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
}
if err := os.RemoveAll(targetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
}
return nil
}
func asString(v any) string {
s, _ := v.(string)
return s
}

View File

@@ -0,0 +1,403 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type archiveSuccessStage struct {
metadata map[string]any
}
func (archiveSuccessStage) Name() string { return "archive" }
func (archiveSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
func (s archiveSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
md := map[string]any{
"stage": "archive",
"uploaded": true,
"current_pointer_written": true,
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
}
for k, v := range s.metadata {
md[k] = v
}
return &stage.StageResult{Metadata: md}, nil
}
type notifyFailStage struct{}
func (notifyFailStage) Name() string { return "notify" }
func (notifyFailStage) Declares() stage.IODecl { return stage.IODecl{} }
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
return nil, errors.New("notify failed")
}
func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
cfg.Pipeline.Workspace.CleanupAfterArchive = false
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, seed.localSourceAudio)
}
func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = false
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertMissing(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, seed.localSourceAudio)
}
func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, cfg.Pipeline.Workspace.Root)
assertExists(t, seed.otherRunDir)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.spoolAudioDir)
}
func TestPostArchiveCleanupBothPolicies(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertMissing(t, seed.spoolAudioDir)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.otherRunDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "archive", err: errors.New("archive failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "stage \"archive\" failed") {
t.Fatalf("executeStages() error = %v, want archive failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
cfg.Pipeline.Archive.UploadRun = boolPtr(false)
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
_, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
t.Fatalf("executeStages() error = %v, want notify failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
cfg, _ := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = false
manifestPath := manifestPathFor(cfg)
store := &manifest.LocalStore{}
m, err := store.Load(context.Background(), manifestPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
m.LocalSpoolDir = filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool")
if err := store.Save(context.Background(), manifestPath, m); err != nil {
t.Fatalf("Save() error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
}
}
func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
cfg, seed, runID := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
cfg.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
}
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
t.Fatalf("executeStages() error = %v, want promotion-missing failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
}
func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
cfg, seed, _ := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
failKey := seed.sessionPrefix + "current/manifest.json"
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
})
if err == nil || !strings.Contains(err.Error(), "current manifest") {
t.Fatalf("executeStages() error = %v, want current-manifest failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
cfg, seed, _ := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
failKey := seed.sessionPrefix + "current/run_id.txt"
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
})
if err == nil || !strings.Contains(err.Error(), "current run pointer") {
t.Fatalf("executeStages() error = %v, want current-run-pointer failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
type cleanupSeed struct {
runWorkDir string
otherRunDir string
spoolAudioDir string
localSourceAudio string
sessionPrefix string
}
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
t.Helper()
cfg := testConfig(t)
cfg.Pipeline.Archive = &config.ArchiveConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
runID := "20260516T010203Z-1a2b3c4d"
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
mustWriteFile(t, localSourceAudio, "source\n")
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.Campaign = cfg.Session.Campaign
seed.RunID = runID
seed.LocalWorkDir = runWorkDir
seed.LocalSpoolDir = spoolAudioDir
seed.S3Bucket = "my-dnd-archive"
seed.S3SessionPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/"
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
store := &manifest.LocalStore{}
if err := os.MkdirAll(filepath.Dir(manifestPathFor(cfg)), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatalf("seed manifest save error = %v", err)
}
return cfg, cleanupSeed{
runWorkDir: runWorkDir,
otherRunDir: otherRunDir,
spoolAudioDir: spoolAudioDir,
localSourceAudio: localSourceAudio,
sessionPrefix: seed.S3SessionPrefix,
}
}
func archiveStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, string) {
t.Helper()
cfg, seed := cleanupFixtureConfig(t)
runID := "20260516T010203Z-1a2b3c4d"
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
cfg.Pipeline.Archive = &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
PromoteArtifacts: []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
},
}
writeArchiveFixtureRunFiles(
t,
seed.runWorkDir,
artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID),
)
store := &manifest.LocalStore{}
seedManifest, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatalf("Load() error = %v", err)
}
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)
if err := store.Save(context.Background(), manifestPathFor(cfg), seedManifest); err != nil {
t.Fatalf("Save() error = %v", err)
}
seed.sessionPrefix = seedManifest.S3SessionPrefix
return cfg, seed, runID
}
func writeArchiveFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
t.Helper()
mustWriteFile(t, filepath.Join(runWorkDir, "prepare", "inputs", "session.yml"), "session_id: 2026-05-03\n")
mustWriteFile(t, filepath.Join(runWorkDir, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "trim", "outputs", "transcripts", "trimmed.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), "{}\n")
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
}
type failKeyStore struct {
delegate *storage.FakeBackend
failKey string
}
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
return s.delegate.List(ctx, prefix)
}
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
return s.delegate.Download(ctx, key, localPath)
}
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
return storage.ObjectInfo{}, errors.New("forced upload failure")
}
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
return s.delegate.Exists(ctx, key)
}
func assertExists(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected path to exist %q: %v", path, err)
}
}
func assertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
}
}

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -17,9 +18,11 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution")
if err := fs.Parse(args); err != nil {
@@ -28,16 +31,18 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("resume: unexpected positional arguments")
}
if sessionPath == "" {
return fmt.Errorf("resume: --session is required")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
cfg, err := config.Load(resolvedPipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("resume: %w", err)
}
@@ -79,7 +84,11 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
}
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
path := manifestPathFor(cfg)
path := artifacts.SessionManifestPathForCampaign(
cfg.Pipeline.Workspace.Root,
cfg.Session.Campaign,
cfg.Session.SessionID,
)
exists, err := fileExists(path)
if err != nil {
return nil, fmt.Errorf("check manifest %q: %w", path, err)

View File

@@ -16,7 +16,7 @@ import (
func TestResumeStartsAfterCompletedStages(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
@@ -25,7 +25,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
if err := store.Save(context.Background(), manifestPath, m); err != nil {
t.Fatalf("save manifest: %v", err)
}
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "raw", "alice.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "autocorrect.yml"), "[]\n")
@@ -52,7 +52,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
func TestResumeNoRemainingStages(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
@@ -81,7 +81,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
@@ -105,8 +105,8 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
@@ -135,8 +135,8 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
func TestRunStageSkipAndForce(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
@@ -166,11 +166,57 @@ func TestRunStageSkipAndForce(t *testing.T) {
}
}
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
store := &manifest.LocalStore{}
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
}
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
t.Fatalf("save manifest: %v", err)
}
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
if err != nil {
t.Fatalf("RunStage(force) error = %v", err)
}
if !strings.Contains(out.String(), "stage=polish executed=1 skipped=0 force=true") {
t.Fatalf("output = %q, want forced polish rerun", out.String())
}
afterForce, err := store.Load(context.Background(), manifestPath)
if err != nil {
t.Fatalf("load manifest after force: %v", err)
}
for _, name := range []string{"normalize", "trim", "analyze", "archive", "notify"} {
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
}
}
out.Reset()
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
if err != nil {
t.Fatalf("Resume() error = %v", err)
}
if !strings.Contains(out.String(), "executed=5 skipped=0") {
t.Fatalf("output = %q, want resume to execute normalize..notify", out.String())
}
}
func TestRunStageTrimExecutes(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer
@@ -198,8 +244,8 @@ func TestRunStageTrimExecutes(t *testing.T) {
func TestRunStageNormalizeExecutes(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
var out bytes.Buffer

View File

@@ -16,9 +16,11 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -27,16 +29,18 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("run: unexpected positional arguments")
}
if sessionPath == "" {
return fmt.Errorf("run: --session is required")
}
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
cfg, err := config.Load(resolvedPipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("run: %w", err)
}

View File

@@ -1,6 +1,8 @@
package app
import (
"time"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
@@ -49,3 +51,43 @@ func firstNonSucceededIndex(stages []stage.Stage, m *manifest.Manifest) int {
}
return len(stages)
}
func canonicalStageNames() []string {
all := stage.All()
out := make([]string, 0, len(all))
for _, s := range all {
if s == nil {
continue
}
out = append(out, s.Name())
}
return out
}
func downstreamStageNames(stageName string) []string {
names := canonicalStageNames()
for i, name := range names {
if name != stageName {
continue
}
return append([]string(nil), names[i+1:]...)
}
return nil
}
func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage string, at time.Time) []string {
if m == nil || m.Stages == nil {
return nil
}
invalidated := make([]string, 0)
for _, downstream := range downstreamStageNames(upstreamStage) {
sr := m.Stages[downstream]
if sr == nil || sr.Status != manifest.StatusSucceeded {
continue
}
m.MarkStageStale(downstream, at, "upstream stage rerun with force")
invalidated = append(invalidated, downstream)
}
return invalidated
}

View File

@@ -1,6 +1,7 @@
package app
import (
"reflect"
"testing"
"time"
@@ -40,3 +41,48 @@ func TestDecideStageActions(t *testing.T) {
t.Fatalf("forced prepare action = %q, want %q", forced[0].Action, stageActionRun)
}
}
func TestDownstreamStageNames(t *testing.T) {
got := downstreamStageNames("polish")
want := []string{"normalize", "trim", "analyze", "archive", "notify"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
}
missing := downstreamStageNames("unknown")
if len(missing) != 0 {
t.Fatalf("downstreamStageNames(unknown) = %#v, want empty", missing)
}
}
func TestInvalidateDownstreamSucceededStages(t *testing.T) {
now := time.Now().UTC()
m := manifest.New("2026-05-03", now)
m.MarkStageSucceeded("prepare", now, nil)
m.MarkStageSucceeded("transcribe", now, nil)
m.MarkStageSucceeded("merge", now, nil)
m.MarkStageSucceeded("polish", now, nil)
m.MarkStageSucceeded("normalize", now, nil)
m.MarkStageSucceeded("trim", now, nil)
m.MarkStageFailed("analyze", now, "analysis failed")
m.MarkStageSucceeded("archive", now, nil)
m.MarkStageSucceeded("notify", now, nil)
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
want := []string{"normalize", "trim", "archive", "notify"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
}
for _, stageName := range want {
if m.Stages[stageName].Status != manifest.StatusStale {
t.Fatalf("%s status = %q, want stale", stageName, m.Stages[stageName].Status)
}
}
if m.Stages["analyze"].Status != manifest.StatusFailed {
t.Fatalf("analyze status = %q, want failed", m.Stages["analyze"].Status)
}
if m.Stages["prepare"].Status != manifest.StatusSucceeded {
t.Fatalf("prepare status = %q, want succeeded", m.Stages["prepare"].Status)
}
}

View File

@@ -16,9 +16,11 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -27,10 +29,6 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 1 {
return fmt.Errorf("run-stage: expected exactly one stage name")
}
if sessionPath == "" {
return fmt.Errorf("run-stage: --session is required")
}
stageName := fs.Arg(0)
stages, err := BuildSingleStagePlan(stageName)
if err != nil {
@@ -41,8 +39,14 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
cfg, err := config.Load(resolvedPipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}

View File

@@ -5,7 +5,6 @@ import (
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
@@ -27,11 +26,13 @@ type RunOptions struct {
}
type RunSummary struct {
SessionID string
ManifestPath string
StageNames []string
Executed []string
Skipped []string
SessionID string
RunID string
ManifestPath string
RunManifestPath string
StageNames []string
Executed []string
Skipped []string
}
func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
@@ -81,17 +82,24 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if env.Storage == nil {
env.Storage = &storage.NoopBackend{}
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
}
env.ObjectStore = objectStore
}
if env.Notifier == nil {
env.Notifier = &notify.NoopSender{}
}
artifactStore := env.ArtifactStore
paths, err := artifactStore.EnsureLayout(cfg.Session.SessionID)
paths, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if err != nil {
return nil, fmt.Errorf("prepare workdir: %w", err)
}
lock, err := artifactStore.AcquireSessionLock(cfg.Session.SessionID)
lock, err := artifactStore.AcquireSessionLockFor(cfg.Session.Campaign, cfg.Session.SessionID)
if err != nil {
return nil, fmt.Errorf("acquire session lock: %w", err)
}
@@ -104,6 +112,42 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if err != nil {
return nil, err
}
runID, err := artifacts.NewRunID()
if err != nil {
return nil, fmt.Errorf("generate run id: %w", err)
}
identityChanged, err := ensureManifestIdentity(cfg, m, runID)
if err != nil {
return nil, fmt.Errorf("initialize manifest identity: %w", err)
}
if identityChanged {
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest identity %q: %w", manifestPath, err)
}
}
runManifestPath := artifacts.SessionRunManifestPathForCampaign(
cfg.Pipeline.Workspace.Root,
cfg.Session.Campaign,
cfg.Session.SessionID,
runID,
)
runManifestStore := &manifest.LocalStore{}
runManifest, err := runManifestStore.CreateRun(
ctx,
cfg.Session.SessionID,
cfg.Session.Campaign,
runID,
opts.Force,
requestedStageNames(stages),
)
if err != nil {
return nil, fmt.Errorf("create run manifest: %w", err)
}
runManifest.SessionManifestPath = manifestPath
syncRunManifestIdentityFromSession(m, runManifest)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err)
}
stageEnv := env
@@ -118,12 +162,23 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if d.Action == stageActionSkip {
skipped = append(skipped, s.Name())
skipAt := nowUTC()
runManifest.SetStageAction(s.Name(), manifest.RunStageActionSkip, skipAt)
runManifest.MarkStageSkipped(s.Name(), skipAt, "already_succeeded")
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err)
}
env.Logger.Info("skipping stage", "stage", s.Name(), "reason", "already_succeeded", "force", opts.Force)
continue
}
executed = append(executed, s.Name())
now := nowUTC()
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
runManifest.MarkStageRunning(s.Name(), now)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
}
m.MarkStageRunning(s.Name(), now)
env.Logger.Info("starting stage", "stage", s.Name())
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
@@ -133,31 +188,65 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
result, err := s.Run(ctx, stageEnv, m)
if err != nil {
m.MarkStageFailed(s.Name(), nowUTC(), err.Error())
failedAt := nowUTC()
m.MarkStageFailed(s.Name(), failedAt, err.Error())
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
}
runManifest.MarkStageFailed(s.Name(), failedAt, err.Error())
syncRunManifestIdentityFromSession(m, runManifest)
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
return nil, fmt.Errorf("stage %q failed (%v) and run-manifest save failed (%v)", s.Name(), err, saveErr)
}
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
}
outputs := mapResultOutputs(result)
m.MarkStageSucceeded(s.Name(), nowUTC(), outputs)
outputs := mapResultOutputs(result, runID)
succeededAt := nowUTC()
m.MarkStageSucceeded(s.Name(), succeededAt, outputs)
applyStageResultToManifest(m, s.Name(), result)
if opts.Force {
invalidateDownstreamSucceededStages(m, s.Name(), succeededAt)
}
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest after stage %q: %w", s.Name(), err)
}
runManifest.MarkStageSucceeded(s.Name(), succeededAt, outputs)
applyStageResultToRunManifest(runManifest, s.Name(), result)
syncRunManifestIdentityFromSession(m, runManifest)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err)
}
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "succeeded", "path", manifestPath)
env.Logger.Info("stage succeeded", "stage", s.Name())
}
if err := runPostArchiveCleanup(ctx, env, manifestPath, m, executed); err != nil {
failedAt := nowUTC()
runManifest.MarkFailed(failedAt, err.Error())
syncRunManifestIdentityFromSession(m, runManifest)
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
return nil, fmt.Errorf("post-archive cleanup failed (%v) and run-manifest save failed (%v)", err, saveErr)
}
return nil, fmt.Errorf("post-archive cleanup: %w", err)
}
completedAt := nowUTC()
runManifest.MarkSucceeded(completedAt)
syncRunManifestIdentityFromSession(m, runManifest)
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
return nil, fmt.Errorf("save final run manifest %q: %w", runManifestPath, err)
}
return &RunSummary{
SessionID: cfg.Session.SessionID,
ManifestPath: manifestPath,
StageNames: runNames,
Executed: executed,
Skipped: skipped,
SessionID: cfg.Session.SessionID,
RunID: runID,
ManifestPath: manifestPath,
RunManifestPath: runManifestPath,
StageNames: runNames,
Executed: executed,
Skipped: skipped,
}, nil
}
@@ -230,7 +319,7 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
}
a := cfg.Pipeline.Audita
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" || len(a.Modules) == 0 || strings.TrimSpace(a.BaseURL) == "" || strings.TrimSpace(a.Model) == "" {
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" {
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
return &audita.NoopRunner{}, nil
}
@@ -247,7 +336,12 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
append([]string(nil), a.Modules...),
a.BaseURL,
a.Model,
a.LLMConcurrency,
a.TranscriptDescription,
a.ConfigPath,
a.OutputSchema,
a.WorkDirRetention,
a.TotalLLMConcurrency,
a.ProposalLLMConcurrency,
a.ValidationModel,
a.ValidationLLMConcurrency,
report,
@@ -292,11 +386,12 @@ func fileExists(path string) (bool, error) {
return false, err
}
func mapResultOutputs(result *stage.StageResult) []manifest.ArtifactRecord {
func mapResultOutputs(result *stage.StageResult, runID string) []manifest.ArtifactRecord {
if result == nil || len(result.Outputs) == 0 {
return nil
}
runID = strings.TrimSpace(runID)
out := make([]manifest.ArtifactRecord, 0, len(result.Outputs))
for _, ref := range result.Outputs {
localPath := ref.AbsolutePath
@@ -304,10 +399,11 @@ func mapResultOutputs(result *stage.StageResult) []manifest.ArtifactRecord {
localPath = ref.RelativePath
}
out = append(out, manifest.ArtifactRecord{
Kind: ref.Kind,
LocalPath: localPath,
RemoteKey: ref.RemoteKey,
Checksum: ref.Checksum,
Kind: ref.Kind,
LocalPath: localPath,
ProducerRunID: runID,
RemoteKey: ref.RemoteKey,
Checksum: ref.Checksum,
})
}
@@ -333,6 +429,132 @@ func applyStageResultToManifest(m *manifest.Manifest, stageName string, result *
}
}
func manifestPathFor(cfg *config.Config) string {
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest, runID string) (bool, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil || m == nil {
return false, nil
}
changed := false
campaign := strings.TrimSpace(cfg.Session.Campaign)
sessionID := strings.TrimSpace(cfg.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
if m.Campaign == "" && campaign != "" {
m.Campaign = campaign
changed = true
}
runID = strings.TrimSpace(runID)
if runID != "" && m.RunID != runID {
m.RunID = runID
changed = true
}
if m.LocalWorkDir == "" && campaign != "" && sessionID != "" && m.RunID != "" {
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID, m.RunID)
changed = true
}
if m.LocalSpoolDir == "" && campaign != "" && sessionID != "" && m.RunID != "" && strings.TrimSpace(cfg.Pipeline.Spool.Root) != "" {
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, campaign, sessionID, m.RunID)
changed = true
}
if cfg.Pipeline.Storage.S3 != nil {
if m.S3Bucket == "" && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
m.S3Bucket = strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket)
changed = true
}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
if m.S3SessionPrefix == "" && sessionPrefix != "" {
m.S3SessionPrefix = sessionPrefix
changed = true
}
runPrefix := artifacts.S3RunPrefix(sessionPrefix, m.RunID)
if m.S3RunPrefix == "" && runPrefix != "" {
m.S3RunPrefix = runPrefix
changed = true
}
}
return changed, nil
}
func requestedStageNames(stages []stage.Stage) []string {
out := make([]string, 0, len(stages))
for _, s := range stages {
if s == nil {
continue
}
out = append(out, s.Name())
}
return out
}
func applyStageResultToRunManifest(m *manifest.RunManifest, stageName string, result *stage.StageResult) {
if m == nil || result == nil {
return
}
sr := m.Stages[stageName]
if sr == nil {
return
}
if len(result.Logs) > 0 {
sr.Logs = append([]string(nil), result.Logs...)
}
if len(result.GeneratedConfigs) > 0 {
sr.GeneratedConfigs = append([]string(nil), result.GeneratedConfigs...)
}
if len(result.Metadata) > 0 {
sr.Metadata = result.Metadata
}
}
func syncRunManifestIdentityFromSession(session *manifest.Manifest, run *manifest.RunManifest) {
if session == nil || run == nil {
return
}
run.Campaign = session.Campaign
run.LocalWorkDir = session.LocalWorkDir
run.LocalSpoolDir = session.LocalSpoolDir
run.S3Bucket = session.S3Bucket
run.S3SessionPrefix = session.S3SessionPrefix
run.S3RunPrefix = session.S3RunPrefix
}
func manifestPathFor(cfg *config.Config) string {
return artifacts.SessionManifestPathForCampaign(
cfg.Pipeline.Workspace.Root,
cfg.Session.Campaign,
cfg.Session.SessionID,
)
}
func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return false
}
stageRequested := func(name string) bool {
for _, s := range stages {
if s != nil && s.Name() == name {
return true
}
}
return false
}
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
return true
}
if !stageRequested("archive") {
return false
}
if cfg.Pipeline.Archive == nil {
return false
}
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
return false
}
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
return false
}
return true
}

View File

@@ -144,6 +144,15 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
}
continue
}
if name == "archive" {
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
}
if sr.Metadata["skipped"] != true {
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
}
continue
}
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
t.Fatalf("stage %q missing placeholder metadata", name)
}
@@ -226,6 +235,56 @@ func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
}
}
func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testing.T) {
cfg := testConfig(t)
manifestPath := manifestPathFor(cfg)
store := &manifest.LocalStore{}
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "archive", "notify"} {
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
}
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
t.Fatalf("Save manifest error = %v", err)
}
runs := 0
stageToRun := countingStage{name: "polish", runs: &runs}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
if err != nil {
t.Fatalf("executeStages() error = %v", err)
}
if runs != 1 {
t.Fatalf("runs = %d, want 1 with force", runs)
}
if len(summary.Executed) != 1 || summary.Executed[0] != "polish" || len(summary.Skipped) != 0 {
t.Fatalf("summary = %#v, want executed polish", summary)
}
loaded, err := store.Load(context.Background(), manifestPath)
if err != nil {
t.Fatalf("Load manifest error = %v", err)
}
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
}
for _, stageName := range []string{"normalize", "trim", "archive", "notify"} {
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
}
}
if loaded.Stages["analyze"] == nil || loaded.Stages["analyze"].Status != manifest.StatusFailed {
t.Fatalf("analyze status = %#v, want preserved failed", loaded.Stages["analyze"])
}
if loaded.Stages["transcribe"] == nil || loaded.Stages["transcribe"].Status != manifest.StatusSucceeded {
t.Fatalf("transcribe status = %#v, want preserved succeeded", loaded.Stages["transcribe"])
}
}
func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
cfg := testConfig(t)
@@ -271,7 +330,14 @@ func TestExecuteStagesLoadsExistingManifest(t *testing.T) {
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
existing.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
audioPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "audio", "alice.flac")
audioPath := filepath.Join(
cfg.Pipeline.Workspace.Root,
"work",
cfg.Session.Campaign,
cfg.Session.SessionID,
"audio",
"alice.flac",
)
if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
@@ -306,6 +372,208 @@ func TestExecuteStagesLoadsExistingManifest(t *testing.T) {
}
}
func TestExecuteStagesCreatesRunManifestPerInvocation(t *testing.T) {
cfg := testConfig(t)
run1, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[0]}, RunOptions{})
if err != nil {
t.Fatalf("first executeStages() error = %v", err)
}
run2, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[0]}, RunOptions{Force: true})
if err != nil {
t.Fatalf("second executeStages() error = %v", err)
}
if run1.RunID == "" || run2.RunID == "" {
t.Fatalf("run ids must be set, got %q and %q", run1.RunID, run2.RunID)
}
if run1.RunID == run2.RunID {
t.Fatalf("expected distinct run ids, got %q", run1.RunID)
}
if run1.RunManifestPath == "" || run2.RunManifestPath == "" {
t.Fatalf("run manifest paths must be set, got %q and %q", run1.RunManifestPath, run2.RunManifestPath)
}
if run1.RunManifestPath == run2.RunManifestPath {
t.Fatalf("expected distinct run manifest paths, got %q", run1.RunManifestPath)
}
for _, path := range []string{run1.RunManifestPath, run2.RunManifestPath} {
if _, statErr := os.Stat(path); statErr != nil {
t.Fatalf("run manifest missing at %q: %v", path, statErr)
}
}
store := &manifest.LocalStore{}
sessionManifest, err := store.Load(context.Background(), run2.ManifestPath)
if err != nil {
t.Fatalf("Load session manifest error = %v", err)
}
if sessionManifest.RunID != run2.RunID {
t.Fatalf("session manifest run_id = %q, want latest run id %q", sessionManifest.RunID, run2.RunID)
}
}
func TestExecuteStagesRunManifestRecordsSkippedStage(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
manifestPath := manifestPathFor(cfg)
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
t.Fatalf("Save manifest error = %v", err)
}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{})
if err != nil {
t.Fatalf("executeStages() error = %v", err)
}
if len(summary.Skipped) != 1 || summary.Skipped[0] != "transcribe" {
t.Fatalf("summary = %#v, want skipped transcribe", summary)
}
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
if err != nil {
t.Fatalf("LoadRun() error = %v", err)
}
sr := runManifest.Stages["transcribe"]
if sr == nil {
t.Fatal("run manifest transcribe stage missing")
}
if sr.Action != manifest.RunStageActionSkip {
t.Fatalf("action = %q, want %q", sr.Action, manifest.RunStageActionSkip)
}
if sr.Status != manifest.StatusSkipped {
t.Fatalf("status = %q, want %q", sr.Status, manifest.StatusSkipped)
}
}
func TestExecuteStagesRunLocalArtifactsAndCanonicalPromotion(t *testing.T) {
cfg := testConfig(t)
stages := []stage.Stage{
BuildFullPlan()[0], // prepare
BuildFullPlan()[1], // transcribe
BuildFullPlan()[2], // merge
BuildFullPlan()[3], // polish
BuildFullPlan()[4], // normalize
BuildFullPlan()[5], // trim
}
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{})
if err != nil {
t.Fatalf("executeStages() error = %v", err)
}
if summary.RunID == "" {
t.Fatal("run id must be set")
}
runRoot := artifacts.SessionRunRootForCampaign(
cfg.Pipeline.Workspace.Root,
cfg.Session.Campaign,
cfg.Session.SessionID,
summary.RunID,
)
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
runLocalChecks := []string{
filepath.Join(runRoot, "transcribe", "outputs", "transcripts", "raw", "alice.json"),
filepath.Join(runRoot, "merge", "logs", "seriatim.stdout.log"),
filepath.Join(runRoot, "polish", "config", "audita.generated.yml"),
filepath.Join(runRoot, "normalize", "logs", "seriatim.normalize.stdout.log"),
filepath.Join(runRoot, "trim", "outputs", "transcripts", "trimmed.json"),
}
for _, p := range runLocalChecks {
if _, statErr := os.Stat(p); statErr != nil {
t.Fatalf("run-local artifact missing at %q: %v", p, statErr)
}
}
canonicalChecks := []string{
filepath.Join(paths.TranscriptsRawDir, "alice.json"),
filepath.Join(paths.TranscriptsDir, "merged.json"),
filepath.Join(paths.TranscriptsDir, "processed.json"),
filepath.Join(paths.TranscriptsDir, "normalized.json"),
filepath.Join(paths.TranscriptsDir, "trimmed.json"),
}
for _, p := range canonicalChecks {
if _, statErr := os.Stat(p); statErr != nil {
t.Fatalf("canonical promoted artifact missing at %q: %v", p, statErr)
}
}
store := &manifest.LocalStore{}
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatalf("Load manifest error = %v", err)
}
if got := sessionManifest.Stages["trim"]; got == nil || len(got.Outputs) == 0 {
t.Fatalf("trim stage outputs missing in session manifest: %#v", got)
}
for _, out := range sessionManifest.Stages["trim"].Outputs {
if strings.Contains(out.LocalPath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("session manifest output should be canonical, got run-local path %q", out.LocalPath)
}
if out.ProducerRunID != summary.RunID {
t.Fatalf("producer_run_id = %q, want %q", out.ProducerRunID, summary.RunID)
}
}
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
if err != nil {
t.Fatalf("LoadRun() error = %v", err)
}
mergeStage := runManifest.Stages["merge"]
if mergeStage == nil || len(mergeStage.Logs) == 0 {
t.Fatalf("merge logs missing in run manifest: %#v", mergeStage)
}
for _, logPath := range mergeStage.Logs {
if !strings.Contains(logPath, filepath.Join("runs", summary.RunID, "merge", "logs")) {
t.Fatalf("run manifest merge log path = %q, want run-local merge logs path", logPath)
}
}
}
func TestExecuteStagesSkippedStagePreservesExistingOutputsProvenance(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
manifestPath := manifestPathFor(cfg)
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{
{
Kind: "transcript_raw",
LocalPath: "transcripts/raw/alice.json",
ProducerRunID: "20260501T000000Z-deadbeef",
},
})
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
t.Fatalf("Save manifest error = %v", err)
}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{})
if err != nil {
t.Fatalf("executeStages() error = %v", err)
}
if len(summary.Skipped) != 1 || summary.Skipped[0] != "transcribe" {
t.Fatalf("summary = %#v, want skipped transcribe", summary)
}
loaded, err := store.Load(context.Background(), manifestPath)
if err != nil {
t.Fatalf("Load manifest error = %v", err)
}
got := loaded.Stages["transcribe"]
if got == nil || len(got.Outputs) != 1 {
t.Fatalf("transcribe outputs = %#v, want one preserved output", got)
}
if got.Outputs[0].ProducerRunID != "20260501T000000Z-deadbeef" {
t.Fatalf("producer_run_id = %q, want preserved value", got.Outputs[0].ProducerRunID)
}
}
func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
cases := []struct {
name string
@@ -315,7 +583,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
{name: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("archive fail")}}},
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
{name: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("notify fail")}}},
}
@@ -332,7 +600,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
tc.env.ArtifactStore = artifactStore
tc.env.ManifestStore = &manifest.LocalStore{}
if tc.name == "transcribe" {
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if ensureErr != nil {
t.Fatalf("EnsureLayout() error = %v", ensureErr)
}
@@ -347,7 +615,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
}
}
if tc.name == "merge" {
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if ensureErr != nil {
t.Fatalf("EnsureLayout() error = %v", ensureErr)
}
@@ -366,7 +634,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
}
}
if tc.name == "polish" {
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if ensureErr != nil {
t.Fatalf("EnsureLayout() error = %v", ensureErr)
}
@@ -378,7 +646,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
}
}
if tc.name == "analyze" {
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if ensureErr != nil {
t.Fatalf("EnsureLayout() error = %v", ensureErr)
}
@@ -400,6 +668,41 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
},
}
}
if tc.name == "archive" {
cfg.Pipeline.Archive = &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
}
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
runID := "20260516T010203Z-0a1b2c3d"
runWorkDir := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, runID)
if err := os.MkdirAll(filepath.Join(runWorkDir, "inputs"), 0o755); err != nil {
t.Fatalf("mkdir archive inputs dir: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "inputs", "session.yml"), []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write archive fixture session.yml: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "manifest.json"), []byte("{}\n"), 0o644); err != nil {
t.Fatalf("write archive fixture manifest.json: %v", err)
}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.Campaign = cfg.Session.Campaign
seed.RunID = runID
seed.LocalWorkDir = runWorkDir
seed.S3Bucket = "my-dnd-archive"
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatalf("seed archive manifest: %v", err)
}
}
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
if runErr == nil {
@@ -430,7 +733,7 @@ func testConfig(t *testing.T) *config.Config {
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -442,6 +745,7 @@ func testConfig(t *testing.T) *config.Config {
SessionPath: sessionPath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
@@ -452,6 +756,55 @@ func testConfig(t *testing.T) *config.Config {
}
}
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace:
root: ` + t.TempDir() + `
whisperx:
transcribe_url: https://example.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
mustWriteFile(t, pipelinePath, pipelineYAML)
mustWriteFile(t, sessionPath, sessionYAML)
cfg, err := config.Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := config.Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
serRunner, err := buildDefaultSeriatimRunner(cfg)
if err != nil {
t.Fatalf("buildDefaultSeriatimRunner() error = %v", err)
}
if _, ok := serRunner.(*seriatim.SubprocessRunner); !ok {
t.Fatalf("seriatim runner type = %T, want *seriatim.SubprocessRunner", serRunner)
}
audRunner, err := buildDefaultAuditaRunner(cfg)
if err != nil {
t.Fatalf("buildDefaultAuditaRunner() error = %v", err)
}
if _, ok := audRunner.(*audita.SubprocessRunner); !ok {
t.Fatalf("audita runner type = %T, want *audita.SubprocessRunner", audRunner)
}
}
func mustWriteFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
@@ -461,3 +814,8 @@ func mustWriteFile(t *testing.T, path, contents string) {
t.Fatalf("WriteFile(%q): %v", path, err)
}
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -0,0 +1,86 @@
package app
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestPlanUsesDiscoveredSessionTemplateWithSessionID(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionTemplate := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionTemplate), 0o644); err != nil {
t.Fatalf("write session template: %v", err)
}
cwd := filepath.Dir(sessionPath)
originalWD, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(cwd); err != nil {
t.Fatalf("Chdir(%q): %v", cwd, err)
}
t.Cleanup(func() { _ = os.Chdir(originalWD) })
var out bytes.Buffer
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session-id", "2026-04-04"}, &out); err != nil {
t.Fatalf("Plan() error = %v", err)
}
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
t.Fatalf("output = %q, want plan output", out.String())
}
}
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
if err != nil {
t.Fatalf("RunStage() error = %v", err)
}
if !strings.Contains(out.String(), "stage=prepare") {
t.Fatalf("output = %q, want stage output", out.String())
}
}
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "searched") {
t.Fatalf("error = %q, want searched paths", err.Error())
}
if !strings.Contains(err.Error(), "pass --session") {
t.Fatalf("error = %q, want explicit-session guidance", err.Error())
}
}

View File

@@ -0,0 +1,49 @@
package app
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func resolveSessionConfigPath(flagValue string) (string, error) {
return resolveSessionConfigPathWithCandidates(flagValue, config.DefaultSessionConfigSearchPaths)
}
func resolveSessionConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
if explicit := strings.TrimSpace(flagValue); explicit != "" {
return explicit, nil
}
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
if path == "" {
continue
}
ordered = append(ordered, path)
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default session config %q: %w", path, err)
}
if len(ordered) == 0 {
return "", fmt.Errorf("no session config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
strings.Join(ordered, ", "),
)
}

View File

@@ -0,0 +1,68 @@
package app
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveSessionConfigPathWithCandidatesExplicitWins(t *testing.T) {
got, err := resolveSessionConfigPathWithCandidates(" ./custom/session.yml ", []string{"./session.yml", "/a", "/b"})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != "./custom/session.yml" {
t.Fatalf("resolved path = %q, want explicit path", got)
}
}
func TestResolveSessionConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(second) {
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
}
}
func TestResolveSessionConfigPathWithCandidatesPrecedence(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(first, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write first default: %v", err)
}
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(first) {
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
}
}
func TestResolveSessionConfigPathWithCandidatesMissing(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "no default session config found") {
t.Fatalf("error = %q, want missing-defaults context", err.Error())
}
if !strings.Contains(err.Error(), "pass --session") {
t.Fatalf("error = %q, want explicit-path guidance", err.Error())
}
}

View File

@@ -63,7 +63,7 @@ func TestExecuteStagesDefaultWiringUsesWhisperXHTTPClient(t *testing.T) {
t.Fatal("audio file payload was empty")
}
outPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "transcripts", "raw", "alice.json")
outPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, "transcripts", "raw", "alice.json")
data, err := os.ReadFile(outPath)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", outPath, err)

View File

@@ -0,0 +1,294 @@
package artifacts
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
const (
ArtifactTranscriptMerged = "narratio.transcript.merged"
ArtifactTranscriptPolished = "narratio.transcript.polished"
ArtifactTranscriptFull = "narratio.transcript.full"
ArtifactTranscriptTrimmed = "narratio.transcript.trimmed"
ArtifactBoundsSession = "narratio.bounds.session"
ArtifactSessionRecap = "narratio.artifact.session_recap"
)
// ErrSessionArtifactNotFound is returned when no readable artifact exists for a known ID.
var ErrSessionArtifactNotFound = errors.New("session artifact not found")
type artifactContentKind string
const (
contentTranscriptJSON artifactContentKind = "transcript_json"
contentJSON artifactContentKind = "json"
contentText artifactContentKind = "text"
)
type artifactSpec struct {
ID string
CanonicalRelPath string
ProducerStage string
OutputKind string
ContentKind artifactContentKind
}
var artifactRegistry = map[string]artifactSpec{
ArtifactTranscriptMerged: {
ID: ArtifactTranscriptMerged,
CanonicalRelPath: "transcripts/merged.json",
ProducerStage: "merge",
OutputKind: "transcript_merged",
ContentKind: contentTranscriptJSON,
},
ArtifactTranscriptPolished: {
ID: ArtifactTranscriptPolished,
CanonicalRelPath: "transcripts/processed.json",
ProducerStage: "polish",
OutputKind: "transcript_processed",
ContentKind: contentTranscriptJSON,
},
ArtifactTranscriptFull: {
ID: ArtifactTranscriptFull,
CanonicalRelPath: "transcripts/normalized.json",
ProducerStage: "normalize",
OutputKind: "transcript_normalized",
ContentKind: contentTranscriptJSON,
},
ArtifactTranscriptTrimmed: {
ID: ArtifactTranscriptTrimmed,
CanonicalRelPath: "transcripts/trimmed.json",
ProducerStage: "trim",
OutputKind: "transcript_trimmed",
ContentKind: contentTranscriptJSON,
},
ArtifactBoundsSession: {
ID: ArtifactBoundsSession,
CanonicalRelPath: "artifacts/session_bounds.json",
ProducerStage: "trim",
OutputKind: "session_bounds",
ContentKind: contentJSON,
},
ArtifactSessionRecap: {
ID: ArtifactSessionRecap,
CanonicalRelPath: "artifacts/session_recap.md",
ProducerStage: "analyze",
OutputKind: "session_recap",
ContentKind: contentText,
},
}
var artifactAliases = map[string]string{
"processed_transcript": ArtifactTranscriptPolished,
"normalized_transcript": ArtifactTranscriptFull,
"trimmed_transcript": ArtifactTranscriptTrimmed,
}
// ResolvedSessionArtifact describes one session-level artifact lookup result.
type ResolvedSessionArtifact struct {
ID string
Path string
ProducerStage string
OutputKind string
ProducerRunID string
Provenance string
}
// SessionArtifactNotFoundError includes context when a known artifact cannot be read.
type SessionArtifactNotFoundError struct {
ArtifactID string
}
func (e *SessionArtifactNotFoundError) Error() string {
return fmt.Sprintf("%s: %q", ErrSessionArtifactNotFound, e.ArtifactID)
}
func (e *SessionArtifactNotFoundError) Unwrap() error {
return ErrSessionArtifactNotFound
}
// NormalizeSessionArtifactSource maps legacy aliases to canonical IDs and validates IDs.
func NormalizeSessionArtifactSource(source string) (string, error) {
normalized := strings.TrimSpace(source)
if normalized == "" {
return "", fmt.Errorf("artifact source is required")
}
if alias, ok := artifactAliases[normalized]; ok {
normalized = alias
}
if _, ok := artifactRegistry[normalized]; !ok {
return "", fmt.Errorf("unsupported artifact source %q", source)
}
return normalized, nil
}
// ResolveSessionArtifact resolves a symbolic source to a readable local session artifact path.
// Resolution order is manifest producer outputs first, then canonical session path fallback.
func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source string) (ResolvedSessionArtifact, error) {
id, err := NormalizeSessionArtifactSource(source)
if err != nil {
return ResolvedSessionArtifact{}, err
}
spec := artifactRegistry[id]
for _, candidate := range manifestArtifactCandidates(paths, m, spec) {
exists, isDir, statErr := pathExists(candidate.Path)
if statErr != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", candidate.Path, statErr)
}
if !exists || isDir {
continue
}
resolved := candidate
resolved.ID = spec.ID
resolved.ProducerStage = spec.ProducerStage
resolved.OutputKind = spec.OutputKind
if err := validateResolvedContent(resolved.Path, spec.ContentKind); err != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", resolved.ID, err)
}
return resolved, nil
}
fallbackPath := filepath.Join(paths.Root, filepath.FromSlash(spec.CanonicalRelPath))
exists, isDir, statErr := pathExists(fallbackPath)
if statErr != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", fallbackPath, statErr)
}
if exists && !isDir {
if err := validateResolvedContent(fallbackPath, spec.ContentKind); err != nil {
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", spec.ID, err)
}
return ResolvedSessionArtifact{
ID: spec.ID,
Path: filepath.Clean(fallbackPath),
ProducerStage: spec.ProducerStage,
OutputKind: spec.OutputKind,
Provenance: "fallback.canonical_path",
}, nil
}
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: spec.ID}
}
func manifestArtifactCandidates(paths SessionPaths, m *manifest.Manifest, spec artifactSpec) []ResolvedSessionArtifact {
if m == nil || len(m.Stages) == 0 || spec.ProducerStage == "" || spec.OutputKind == "" {
return nil
}
sr := m.Stages[spec.ProducerStage]
if sr == nil {
return nil
}
candidates := make([]ResolvedSessionArtifact, 0, len(sr.Outputs))
for _, out := range sr.Outputs {
if strings.TrimSpace(out.Kind) != spec.OutputKind {
continue
}
if strings.TrimSpace(out.LocalPath) == "" {
continue
}
resolved := filepath.Clean(ResolveSessionLocalPathForRead(paths, out.LocalPath))
if resolved == "" {
continue
}
candidates = append(candidates, ResolvedSessionArtifact{
Path: resolved,
ProducerRunID: strings.TrimSpace(out.ProducerRunID),
Provenance: "manifest." + spec.ProducerStage + ".outputs",
})
}
return dedupeResolvedArtifacts(candidates)
}
func dedupeResolvedArtifacts(values []ResolvedSessionArtifact) []ResolvedSessionArtifact {
seen := map[string]struct{}{}
out := make([]ResolvedSessionArtifact, 0, len(values))
for _, value := range values {
key := filepath.Clean(strings.TrimSpace(value.Path))
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
value.Path = key
out = append(out, value)
}
return out
}
func pathExists(path string) (exists bool, isDir bool, err error) {
info, err := os.Stat(path)
if err == nil {
return true, info.IsDir(), nil
}
if errors.Is(err, os.ErrNotExist) {
return false, false, nil
}
return false, false, err
}
func validateResolvedContent(path string, kind artifactContentKind) error {
switch kind {
case contentTranscriptJSON:
return validateTranscriptSegmentsJSON(path)
case contentJSON:
return validateJSONContent(path)
case contentText:
return validateNonEmptyContent(path)
default:
return fmt.Errorf("unsupported content kind %q", kind)
}
}
func validateTranscriptSegmentsJSON(path string) error {
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("read file: %w", err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
return fmt.Errorf("decode json: %w", err)
}
segments, ok := payload["segments"]
if !ok {
return fmt.Errorf("top-level segments is required")
}
if _, ok := segments.([]any); !ok {
return fmt.Errorf("top-level segments must be an array")
}
return nil
}
func validateJSONContent(path string) error {
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("read file: %w", err)
}
var payload any
if err := json.Unmarshal(data, &payload); err != nil {
return fmt.Errorf("decode json: %w", err)
}
return nil
}
func validateNonEmptyContent(path string) error {
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("stat file: %w", err)
}
if info.IsDir() {
return fmt.Errorf("path is a directory")
}
if info.Size() <= 0 {
return fmt.Errorf("file is empty")
}
return nil
}

View File

@@ -0,0 +1,139 @@
package artifacts
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestNormalizeSessionArtifactSource(t *testing.T) {
tests := []struct {
name string
source string
wantID string
wantErr string
}{
{name: "legacy alias processed", source: "processed_transcript", wantID: ArtifactTranscriptPolished},
{name: "legacy alias normalized", source: "normalized_transcript", wantID: ArtifactTranscriptFull},
{name: "legacy alias trimmed", source: "trimmed_transcript", wantID: ArtifactTranscriptTrimmed},
{name: "canonical", source: ArtifactTranscriptTrimmed, wantID: ArtifactTranscriptTrimmed},
{name: "unsupported", source: "narratio.unknown", wantErr: "unsupported artifact source"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeSessionArtifactSource(tt.source)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("NormalizeSessionArtifactSource() error = %v, want contains %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("NormalizeSessionArtifactSource() error = %v", err)
}
if got != tt.wantID {
t.Fatalf("NormalizeSessionArtifactSource() = %q, want %q", got, tt.wantID)
}
})
}
}
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
if err := os.MkdirAll(paths.ArtifactsDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
if err := os.WriteFile(manifestPath, []byte(`{"segments":[]}`), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
canonicalPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(canonicalPath, []byte(`{"segments":[{"id":123}]}`), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
m := manifest.New("session", time.Now().UTC())
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
{Kind: "transcript_normalized", LocalPath: manifestPath, ProducerRunID: "run-123"},
})
resolved, err := ResolveSessionArtifact(paths, m, "normalized_transcript")
if err != nil {
t.Fatalf("ResolveSessionArtifact() error = %v", err)
}
if resolved.Path != manifestPath {
t.Fatalf("resolved path = %q, want %q", resolved.Path, manifestPath)
}
if resolved.Provenance != "manifest.normalize.outputs" {
t.Fatalf("provenance = %q, want manifest.normalize.outputs", resolved.Provenance)
}
if resolved.ProducerRunID != "run-123" {
t.Fatalf("producer run id = %q, want run-123", resolved.ProducerRunID)
}
}
func TestResolveSessionArtifactFallsBackToCanonicalPath(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
canonicalPath := filepath.Join(paths.TranscriptsDir, "trimmed.json")
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(canonicalPath, []byte(`{"segments":[]}`), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
resolved, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptTrimmed)
if err != nil {
t.Fatalf("ResolveSessionArtifact() error = %v", err)
}
if resolved.Path != canonicalPath {
t.Fatalf("resolved path = %q, want %q", resolved.Path, canonicalPath)
}
if resolved.Provenance != "fallback.canonical_path" {
t.Fatalf("provenance = %q, want fallback.canonical_path", resolved.Provenance)
}
}
func TestResolveSessionArtifactMissingReturnsTypedError(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
_, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptTrimmed)
if err == nil {
t.Fatal("expected error, got nil")
}
if !errors.Is(err, ErrSessionArtifactNotFound) {
t.Fatalf("errors.Is(err, ErrSessionArtifactNotFound) = false; err=%v", err)
}
}
func TestResolveSessionArtifactValidatesTranscriptShape(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "campaign", "session")
canonicalPath := filepath.Join(paths.TranscriptsDir, "processed.json")
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := os.WriteFile(canonicalPath, []byte(`{"not_segments":[]}`), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
_, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptPolished)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "top-level segments is required") {
t.Fatalf("error = %q, want segments validation error", err.Error())
}
}

View File

@@ -30,13 +30,13 @@ func NewLocalStore(workspaceRoot string) *LocalStore {
return &LocalStore{WorkspaceRoot: workspaceRoot}
}
// SessionPaths resolves canonical paths for a session workdir.
func (s *LocalStore) SessionPaths(sessionID string) SessionPaths {
return buildSessionPaths(s.WorkspaceRoot, sessionID)
// SessionPathsFor resolves canonical campaign-aware paths for a session workdir.
func (s *LocalStore) SessionPathsFor(campaign, sessionID string) SessionPaths {
return buildSessionPaths(s.WorkspaceRoot, campaign, sessionID)
}
// EnsureLayout creates and verifies the canonical session workdir directory layout.
func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
// EnsureLayoutFor creates and verifies campaign-aware session layout.
func (s *LocalStore) EnsureLayoutFor(campaign, sessionID string) (SessionPaths, error) {
if strings.TrimSpace(s.WorkspaceRoot) == "" {
return SessionPaths{}, fmt.Errorf("workspace root is required")
}
@@ -44,7 +44,22 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
return SessionPaths{}, fmt.Errorf("sessionID is required")
}
paths := s.SessionPaths(sessionID)
campaign = strings.TrimSpace(campaign)
if campaign == "" {
return SessionPaths{}, fmt.Errorf("campaign is required")
}
return s.ensureLayout(s.SessionPathsFor(campaign, sessionID))
}
func (s *LocalStore) ensureLayout(paths SessionPaths) (SessionPaths, error) {
if strings.TrimSpace(s.WorkspaceRoot) == "" {
return SessionPaths{}, fmt.Errorf("workspace root is required")
}
if strings.TrimSpace(paths.SessionID) == "" {
return SessionPaths{}, fmt.Errorf("sessionID is required")
}
dirs := []string{
paths.Root,
paths.InputsDir,
@@ -53,8 +68,11 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
paths.TranscriptsRawDir,
paths.TranscriptsTrimmedDir,
paths.ArtifactsDir,
paths.ReportsDir,
paths.ConfigDir,
paths.LogsDir,
paths.CurrentDir,
paths.RunsDir,
}
for _, dir := range dirs {
@@ -66,13 +84,16 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
return paths, nil
}
// CopyInput copies an input file into the session workdir under destRelativePath.
func (s *LocalStore) CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error) {
paths, err := s.EnsureLayout(sessionID)
// CopyInputFor copies an input file into the campaign-aware session workdir under destRelativePath.
func (s *LocalStore) CopyInputFor(campaign, sessionID, srcPath, destRelativePath string) (Ref, error) {
paths, err := s.EnsureLayoutFor(campaign, sessionID)
if err != nil {
return Ref{}, err
}
return s.copyInputWithPaths(paths, sessionID, srcPath, destRelativePath)
}
func (s *LocalStore) copyInputWithPaths(paths SessionPaths, sessionID, srcPath, destRelativePath string) (Ref, error) {
destAbs, err := resolveInRoot(paths.Root, destRelativePath)
if err != nil {
return Ref{}, fmt.Errorf("copy input: %w", err)
@@ -173,13 +194,16 @@ func (s *LocalStore) Checksum(path string) (string, error) {
return digest, nil
}
// AcquireSessionLock acquires an exclusive lock file for a session workdir.
func (s *LocalStore) AcquireSessionLock(sessionID string) (*LockHandle, error) {
paths, err := s.EnsureLayout(sessionID)
// AcquireSessionLockFor acquires an exclusive lock file for a campaign/session workdir.
func (s *LocalStore) AcquireSessionLockFor(campaign, sessionID string) (*LockHandle, error) {
paths, err := s.EnsureLayoutFor(campaign, sessionID)
if err != nil {
return nil, err
}
return s.acquireSessionLockForPaths(paths)
}
func (s *LocalStore) acquireSessionLockForPaths(paths SessionPaths) (*LockHandle, error) {
f, err := os.OpenFile(paths.LockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
if err != nil {
if errors.Is(err, os.ErrExist) {

View File

@@ -10,9 +10,9 @@ import (
func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
store := NewLocalStore(t.TempDir())
paths, err := store.EnsureLayout("session-1")
paths, err := store.EnsureLayoutFor("sample-campaign", "session-1")
if err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
t.Fatalf("EnsureLayoutFor() error = %v", err)
}
checkDirExists(t, paths.Root)
@@ -22,8 +22,11 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
checkDirExists(t, paths.TranscriptsRawDir)
checkDirExists(t, paths.TranscriptsTrimmedDir)
checkDirExists(t, paths.ArtifactsDir)
checkDirExists(t, paths.ReportsDir)
checkDirExists(t, paths.ConfigDir)
checkDirExists(t, paths.LogsDir)
checkDirExists(t, paths.CurrentDir)
checkDirExists(t, paths.RunsDir)
if filepath.Base(paths.ManifestPath) != "manifest.json" {
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
@@ -33,6 +36,17 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
}
}
func TestEnsureLayoutForRequiresCampaign(t *testing.T) {
store := NewLocalStore(t.TempDir())
_, err := store.EnsureLayoutFor("", "session-1")
if err == nil {
t.Fatal("expected campaign-required error, got nil")
}
if !strings.Contains(err.Error(), "campaign is required") {
t.Fatalf("error = %v, want campaign-required error", err)
}
}
func TestChecksumCalculation(t *testing.T) {
store := NewLocalStore(t.TempDir())
path := filepath.Join(t.TempDir(), "sample.txt")
@@ -53,9 +67,9 @@ func TestChecksumCalculation(t *testing.T) {
func TestLockAcquireRelease(t *testing.T) {
store := NewLocalStore(t.TempDir())
lock, err := store.AcquireSessionLock("session-1")
lock, err := store.AcquireSessionLockFor("sample-campaign", "session-1")
if err != nil {
t.Fatalf("AcquireSessionLock() error = %v", err)
t.Fatalf("AcquireSessionLockFor() error = %v", err)
}
exists, err := store.Exists(lock.path)
@@ -81,15 +95,15 @@ func TestLockAcquireRelease(t *testing.T) {
func TestLockConflict(t *testing.T) {
store := NewLocalStore(t.TempDir())
lock1, err := store.AcquireSessionLock("session-1")
lock1, err := store.AcquireSessionLockFor("sample-campaign", "session-1")
if err != nil {
t.Fatalf("first AcquireSessionLock() error = %v", err)
t.Fatalf("first AcquireSessionLockFor() error = %v", err)
}
defer func() {
_ = store.ReleaseSessionLock(lock1)
}()
_, err = store.AcquireSessionLock("session-1")
_, err = store.AcquireSessionLockFor("sample-campaign", "session-1")
if err == nil {
t.Fatal("expected lock conflict error, got nil")
}
@@ -138,9 +152,9 @@ func TestCopyInput(t *testing.T) {
t.Fatalf("WriteFile() error = %v", err)
}
ref, err := store.CopyInput("session-1", srcPath, "inputs/speakers.yml")
ref, err := store.CopyInputFor("sample-campaign", "session-1", srcPath, "inputs/speakers.yml")
if err != nil {
t.Fatalf("CopyInput() error = %v", err)
t.Fatalf("CopyInputFor() error = %v", err)
}
if ref.Kind != "input" {

View File

@@ -1,10 +1,16 @@
package artifacts
import "path/filepath"
import (
"path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// SessionPaths contains canonical local paths for one session work directory.
type SessionPaths struct {
WorkspaceRoot string
CampaignID string
SessionID string
Root string
InputsDir string
AudioDir string
@@ -12,32 +18,73 @@ type SessionPaths struct {
TranscriptsRawDir string
TranscriptsTrimmedDir string
ArtifactsDir string
ReportsDir string
ConfigDir string
LogsDir string
CurrentDir string
RunsDir string
ManifestPath string
LockPath string
}
// SessionWorkDir returns the work directory for one session.
func SessionWorkDir(rootDir, sessionID string) string {
return filepath.Join(rootDir, "work", sessionID)
// SessionWorkDirForCampaign returns the canonical campaign-aware work directory for one session.
func SessionWorkDirForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(rootDir, config.PathWorkDirSegment, campaign, sessionID)
}
func buildSessionPaths(workspaceRoot, sessionID string) SessionPaths {
root := SessionWorkDir(workspaceRoot, sessionID)
transcripts := filepath.Join(root, "transcripts")
// SessionManifestPathForCampaign returns the canonical session manifest path.
func SessionManifestPathForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathManifestFile)
}
// SessionRunsDirForCampaign returns the canonical runs directory for one session.
func SessionRunsDirForCampaign(rootDir, campaign, sessionID string) string {
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathRunsDirSegment)
}
// SessionRunRootForCampaign returns the canonical run root under runs/{run_id}.
func SessionRunRootForCampaign(rootDir, campaign, sessionID, runID string) string {
return filepath.Join(SessionRunsDirForCampaign(rootDir, campaign, sessionID), runID)
}
// SessionRunManifestPathForCampaign returns the canonical run manifest path under runs/{run_id}/manifest.json.
func SessionRunManifestPathForCampaign(rootDir, campaign, sessionID, runID string) string {
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), config.PathManifestFile)
}
// SessionRunStageDirForCampaign returns the canonical stage directory under runs/{run_id}/{stage}.
func SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, stageName string) string {
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), stageName)
}
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
}
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
root := SessionWorkDirForCampaign(workspaceRoot, campaign, sessionID)
return buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root)
}
func buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root string) SessionPaths {
return SessionPaths{
WorkspaceRoot: workspaceRoot,
CampaignID: campaign,
SessionID: sessionID,
Root: root,
InputsDir: filepath.Join(root, "inputs"),
AudioDir: filepath.Join(root, "audio"),
TranscriptsDir: transcripts,
TranscriptsRawDir: filepath.Join(transcripts, "raw"),
TranscriptsTrimmedDir: filepath.Join(transcripts, "trimmed"),
ArtifactsDir: filepath.Join(root, "artifacts"),
ConfigDir: filepath.Join(root, "config"),
LogsDir: filepath.Join(root, "logs"),
ManifestPath: filepath.Join(root, "manifest.json"),
LockPath: filepath.Join(root, ".lock"),
InputsDir: filepath.Join(root, config.PathInputsDirSegment),
AudioDir: filepath.Join(root, config.PathAudioDirSegment),
TranscriptsDir: filepath.Join(root, config.PathTranscriptsSegment),
TranscriptsRawDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsRaw)),
TranscriptsTrimmedDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsTrimmed)),
ArtifactsDir: filepath.Join(root, config.PathArtifactsDirSegment),
ReportsDir: filepath.Join(root, config.PathReportsDirSegment),
ConfigDir: filepath.Join(root, config.PathConfigDirSegment),
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
CurrentDir: filepath.Join(root, config.PathCurrentDirSegment),
RunsDir: filepath.Join(root, config.PathRunsDirSegment),
ManifestPath: filepath.Join(root, config.PathManifestFile),
LockPath: filepath.Join(root, config.PathLockFile),
}
}

View File

@@ -0,0 +1,59 @@
package artifacts
import (
"path/filepath"
"testing"
)
func TestSessionWorkDirForCampaign(t *testing.T) {
root := "/tmp/workspace"
got := SessionWorkDirForCampaign(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "work", "forsaken", "2026-04-19")
if got != want {
t.Fatalf("SessionWorkDirForCampaign() = %q, want %q", got, want)
}
}
func TestSessionManifestPathForCampaign(t *testing.T) {
root := "/tmp/workspace"
got := SessionManifestPathForCampaign(root, "forsaken", "2026-04-19")
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "manifest.json")
if got != want {
t.Fatalf("SessionManifestPathForCampaign() = %q, want %q", got, want)
}
}
func TestSessionRunRootAndStageDirForCampaign(t *testing.T) {
root := "/tmp/workspace"
runID := "20260515T031522Z-a1b2c3d4"
runRoot := SessionRunRootForCampaign(root, "forsaken", "2026-04-19", runID)
wantRoot := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID)
if runRoot != wantRoot {
t.Fatalf("SessionRunRootForCampaign() = %q, want %q", runRoot, wantRoot)
}
stageDir := SessionRunStageDirForCampaign(root, "forsaken", "2026-04-19", runID, "transcribe")
wantStage := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID, "transcribe")
if stageDir != wantStage {
t.Fatalf("SessionRunStageDirForCampaign() = %q, want %q", stageDir, wantStage)
}
}
func TestSessionRunManifestPathForCampaign(t *testing.T) {
root := "/tmp/workspace"
runID := "20260515T031522Z-a1b2c3d4"
got := SessionRunManifestPathForCampaign(root, "forsaken", "2026-04-19", runID)
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID, "manifest.json")
if got != want {
t.Fatalf("SessionRunManifestPathForCampaign() = %q, want %q", got, want)
}
}
func TestSessionSpoolAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
want := filepath.Join(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4", "audio")
if got != want {
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
}
}

View File

@@ -8,7 +8,7 @@ import (
func TestResolveSessionLocalPathForRead(t *testing.T) {
workspace := t.TempDir()
paths := buildSessionPaths(workspace, "s-1")
paths := buildSessionPaths(workspace, "sample-campaign", "s-1")
if err := os.MkdirAll(paths.TranscriptsRawDir, 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
@@ -41,7 +41,7 @@ func TestResolveSessionLocalPathForReadRelativeWorkspaceRootQualifiedPath(t *tes
t.Fatalf("Rel() error = %v", err)
}
paths := buildSessionPaths(workspaceRel, "s-1")
paths := buildSessionPaths(workspaceRel, "sample-campaign", "s-1")
target := filepath.Join(paths.TranscriptsRawDir, "alice.json")
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
@@ -50,7 +50,7 @@ func TestResolveSessionLocalPathForReadRelativeWorkspaceRootQualifiedPath(t *tes
t.Fatalf("WriteFile() error = %v", err)
}
manifestPath := filepath.Join(workspaceRel, "work", "s-1", "transcripts", "raw", "alice.json")
manifestPath := filepath.Join(workspaceRel, "work", "sample-campaign", "s-1", "transcripts", "raw", "alice.json")
got := ResolveSessionLocalPathForRead(paths, manifestPath)
if got != filepath.Clean(manifestPath) {
t.Fatalf("resolution = %q, want %q", got, filepath.Clean(manifestPath))

View File

@@ -0,0 +1,30 @@
package artifacts
import (
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"time"
)
// NewRunID returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx.
func NewRunID() (string, error) {
return NewRunIDWith(time.Now().UTC(), rand.Reader)
}
// NewRunIDWith returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx
// using an injected timestamp and randomness source.
func NewRunIDWith(now time.Time, random io.Reader) (string, error) {
if random == nil {
random = rand.Reader
}
var suffix [4]byte
if _, err := io.ReadFull(random, suffix[:]); err != nil {
return "", fmt.Errorf("generate run id random suffix: %w", err)
}
ts := now.UTC().Format("20060102T150405Z")
return ts + "-" + hex.EncodeToString(suffix[:]), nil
}

View File

@@ -0,0 +1,37 @@
package artifacts
import (
"bytes"
"regexp"
"strings"
"testing"
"time"
)
func TestNewRunIDWithFormat(t *testing.T) {
now := time.Date(2026, 5, 15, 3, 15, 22, 0, time.UTC)
random := bytes.NewReader([]byte{0xa1, 0xb2, 0xc3, 0xd4})
runID, err := NewRunIDWith(now, random)
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
if runID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("runID = %q, want %q", runID, "20260515T031522Z-a1b2c3d4")
}
}
func TestNewRunIDWithShape(t *testing.T) {
runID, err := NewRunIDWith(time.Now().UTC(), bytes.NewReader([]byte{0x01, 0x02, 0x03, 0x04}))
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
pattern := regexp.MustCompile(`^\d{8}T\d{6}Z-[0-9a-f]{8}$`)
if !pattern.MatchString(runID) {
t.Fatalf("runID = %q, want pattern %q", runID, pattern.String())
}
suffix := runID[len(runID)-8:]
if strings.ToLower(suffix) != suffix {
t.Fatalf("runID suffix = %q, want lowercase", suffix)
}
}

View File

@@ -11,19 +11,19 @@ type S3Store struct {
Prefix string
}
// SessionPaths is not implemented for S3-backed storage.
func (s *S3Store) SessionPaths(_ string) SessionPaths {
// SessionPathsFor is not implemented for S3-backed storage.
func (s *S3Store) SessionPathsFor(_, _ string) SessionPaths {
return SessionPaths{}
}
// EnsureLayout returns a not-yet-implemented error in the scaffold.
func (s *S3Store) EnsureLayout(_ string) (SessionPaths, error) {
return SessionPaths{}, fmt.Errorf("artifacts s3 ensure layout: not yet implemented")
// EnsureLayoutFor returns a not-yet-implemented error in the scaffold.
func (s *S3Store) EnsureLayoutFor(_, _ string) (SessionPaths, error) {
return SessionPaths{}, fmt.Errorf("artifacts s3 ensure layout for campaign/session: not yet implemented")
}
// CopyInput returns a not-yet-implemented error in the scaffold.
func (s *S3Store) CopyInput(_, _, _ string) (Ref, error) {
return Ref{}, fmt.Errorf("artifacts s3 copy input: not yet implemented")
// CopyInputFor returns a not-yet-implemented error in the scaffold.
func (s *S3Store) CopyInputFor(_, _, _, _ string) (Ref, error) {
return Ref{}, fmt.Errorf("artifacts s3 copy input for campaign/session: not yet implemented")
}
// Exists returns a not-yet-implemented error in the scaffold.
@@ -46,9 +46,9 @@ func (s *S3Store) Checksum(_ string) (string, error) {
return "", fmt.Errorf("artifacts s3 checksum: not yet implemented")
}
// AcquireSessionLock returns a not-yet-implemented error in the scaffold.
func (s *S3Store) AcquireSessionLock(_ string) (*LockHandle, error) {
return nil, fmt.Errorf("artifacts s3 acquire lock: not yet implemented")
// AcquireSessionLockFor returns a not-yet-implemented error in the scaffold.
func (s *S3Store) AcquireSessionLockFor(_, _ string) (*LockHandle, error) {
return nil, fmt.Errorf("artifacts s3 acquire lock for campaign/session: not yet implemented")
}
// ReleaseSessionLock returns a not-yet-implemented error in the scaffold.

View File

@@ -0,0 +1,75 @@
package artifacts
import (
"path"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// S3SessionPrefix builds the canonical S3 session prefix.
// Format: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
func S3SessionPrefix(rootPrefix, campaign, sessionID string) string {
prefix := path.Join(
cleanS3PathPart(rootPrefix),
config.S3CampaignsSegment,
cleanS3PathPart(campaign),
config.S3SessionsSegment,
cleanS3PathPart(sessionID),
)
return ensureS3TrailingSlash(prefix)
}
// S3RunPrefix builds the canonical S3 run prefix.
// Format: {session_prefix}/runs/{run_id}/
func S3RunPrefix(sessionPrefix, runID string) string {
prefix := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3RunsSegment, cleanS3PathPart(runID))
return ensureS3TrailingSlash(prefix)
}
// S3AudioPrefix builds the session audio prefix from configured audio_s3.prefix.
// Format: {session_prefix}/{audio_s3.prefix}
func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
key := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(audioPrefix))
return ensureS3TrailingSlash(key)
}
// S3CurrentManifestKey returns the current manifest pointer key.
// Format: {session_prefix}/current/manifest.json
func S3CurrentManifestKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3ManifestFile)
}
// S3CurrentRunPointerKey returns the current run pointer key.
// Format: {session_prefix}/current/run_id.txt
func S3CurrentRunPointerKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3RunIDFile)
}
// S3PromotedArtifactKey returns the destination key for one promoted artifact.
// Format: {session_prefix}/{promotion.to}
func S3PromotedArtifactKey(sessionPrefix, to string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
}
// S3RunRelativeDestinationKey returns a run-scoped key for a workdir-relative path.
// Format: {run_prefix}/{relative_workdir_path}
func S3RunRelativeDestinationKey(runPrefix, relativeWorkdirPath string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(runPrefix), "/"), cleanS3Key(relativeWorkdirPath))
}
func ensureS3TrailingSlash(v string) string {
key := cleanS3Key(v)
if key == "" {
return ""
}
return strings.TrimSuffix(key, "/") + "/"
}
func cleanS3PathPart(v string) string {
return strings.Trim(strings.ReplaceAll(strings.TrimSpace(v), "\\", "/"), "/")
}
func cleanS3Key(v string) string {
return strings.ReplaceAll(strings.TrimSpace(v), "\\", "/")
}

View File

@@ -0,0 +1,45 @@
package artifacts
import (
"strings"
"testing"
)
func TestS3KeyConstruction(t *testing.T) {
runID := "20260515T031522Z-a1b2c3d4"
sessionPrefix := S3SessionPrefix("dnd", "forsaken", "2026-04-19")
if sessionPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/" {
t.Fatalf("sessionPrefix = %q", sessionPrefix)
}
audioPrefix := S3AudioPrefix(sessionPrefix, "audio/")
if audioPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/audio/" {
t.Fatalf("audioPrefix = %q", audioPrefix)
}
runPrefix := S3RunPrefix(sessionPrefix, runID)
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
if runPrefix != wantRunPrefix {
t.Fatalf("runPrefix = %q, want %q", runPrefix, wantRunPrefix)
}
runPointer := S3CurrentRunPointerKey(sessionPrefix)
if runPointer != "dnd/campaigns/forsaken/sessions/2026-04-19/current/run_id.txt" {
t.Fatalf("run pointer key = %q", runPointer)
}
manifestKey := S3CurrentManifestKey(sessionPrefix)
if manifestKey != "dnd/campaigns/forsaken/sessions/2026-04-19/current/manifest.json" {
t.Fatalf("manifest key = %q", manifestKey)
}
promoted := S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/trimmed.json" {
t.Fatalf("promoted key = %q", promoted)
}
runRelative := S3RunRelativeDestinationKey(runPrefix, `logs\whisperx.stdout.log`)
if !strings.HasSuffix(runRelative, "/logs/whisperx.stdout.log") {
t.Fatalf("runRelative key = %q, want normalized forward slashes", runRelative)
}
}

View File

@@ -15,13 +15,13 @@ type Ref struct {
// Store is the local artifact/workdir abstraction used by orchestration code.
type Store interface {
SessionPaths(sessionID string) SessionPaths
EnsureLayout(sessionID string) (SessionPaths, error)
CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error)
SessionPathsFor(campaign, sessionID string) SessionPaths
EnsureLayoutFor(campaign, sessionID string) (SessionPaths, error)
CopyInputFor(campaign, sessionID, srcPath, destRelativePath string) (Ref, error)
Exists(path string) (bool, error)
ExistsRef(ref Ref) (bool, error)
WriteFileAtomic(path string, data []byte, perm os.FileMode) error
Checksum(path string) (string, error)
AcquireSessionLock(sessionID string) (*LockHandle, error)
AcquireSessionLockFor(campaign, sessionID string) (*LockHandle, error)
ReleaseSessionLock(lock *LockHandle) error
}

View File

@@ -12,6 +12,8 @@ type Config struct {
type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"`
Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"`
Archive *ArchiveConfig `yaml:"archive"`
Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim"`
@@ -34,7 +36,8 @@ type SessionConfig struct {
// WorkspaceConfig configures local workspace behavior.
type WorkspaceConfig struct {
Root string `yaml:"root"`
Root string `yaml:"root"`
CleanupAfterArchive bool `yaml:"cleanup_after_archive"`
}
// SecretsConfig configures optional local filesystem secret loading.
@@ -44,9 +47,41 @@ type SecretsConfig struct {
// StorageConfig configures storage backends and related parameters.
type StorageConfig struct {
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
S3 *StorageS3Config `yaml:"s3"`
}
// StorageS3Config configures S3 storage coordinates.
type StorageS3Config struct {
Bucket string `yaml:"bucket"`
RootPrefix string `yaml:"root_prefix"`
Region string `yaml:"region"`
Endpoint string `yaml:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style"`
AccessKeyIDEnv string `yaml:"access_key_id_env"`
SecretKeyEnv string `yaml:"secret_access_key_env"`
}
// SpoolConfig configures local spool storage for staged data.
type SpoolConfig struct {
Root string `yaml:"root"`
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
}
// ArchiveConfig configures archive behavior and artifact promotions.
type ArchiveConfig struct {
Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"`
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
}
// ArchivePromotionRule configures one artifact promotion mapping.
type ArchivePromotionRule struct {
From string `yaml:"from"`
To string `yaml:"to"`
Required *bool `yaml:"required"`
}
// WhisperXConfig configures WhisperX adapter settings.
@@ -85,9 +120,14 @@ type AuditaConfig struct {
Modules []string `yaml:"modules"`
BaseURL string `yaml:"base_url"`
Model string `yaml:"model"`
LLMConcurrency *int `yaml:"llm_concurrency"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description"`
ConfigPath string `yaml:"config_path"`
OutputSchema string `yaml:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention"`
Report *bool `yaml:"report"`
}
@@ -175,9 +215,15 @@ type ArtifactSettings struct {
// SessionInputsConfig contains per-session input references.
type SessionInputsConfig struct {
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
}
// SessionAudioS3Input configures S3 session-audio input discovery.
type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"`
}

View File

@@ -5,8 +5,77 @@ package config
const (
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
DefaultSessionConfigPathLocal = "./session.yml"
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
DefaultS3SecretAccessKeyEnv = "OBJECT_STORAGE_KEY"
DefaultStorageS3RootPrefix = "dnd"
DefaultSpoolRoot = "/var/spool/narratio"
DefaultWhisperXLanguage = "en"
DefaultWhisperXTimeout = "30m"
DefaultWhisperXRetryDelay = "2s"
DefaultWhisperXConcurrency = 2
DefaultWhisperXRetries = 3
DefaultSeriatimBinary = "seriatim"
DefaultSeriatimTimeout = "10m"
DefaultSeriatimOutputSchema = "seriatim-intermediate"
DefaultSeriatimCoalesceGap = 3.0
DefaultSeriatimReport = true
DefaultAuditaBinary = "audita"
DefaultAuditaTimeout = "3h"
DefaultAuditaReport = true
DefaultScriptoriumBinary = "scriptorium"
DefaultScriptoriumTimeout = "10m"
DefaultTrimBoundsTimeout = "10m"
DefaultTrimSeriatimReport = false
DefaultNormalizeOutputPath = "transcripts/normalized.json"
DefaultNormalizeOutputSchema = "seriatim-intermediate"
DefaultNormalizeReport = true
DefaultArchiveEnabled = true
DefaultArchiveUploadRun = true
PathWorkDirSegment = "work"
PathInputsDirSegment = "inputs"
PathAudioDirSegment = "audio"
PathTranscriptsSegment = "transcripts"
PathTranscriptsRaw = "transcripts/raw"
PathTranscriptsTrimmed = "transcripts/trimmed"
PathArtifactsDirSegment = "artifacts"
PathReportsDirSegment = "reports"
PathConfigDirSegment = "config"
PathLogsDirSegment = "logs"
PathCurrentDirSegment = "current"
PathRunsDirSegment = "runs"
PathManifestFile = "manifest.json"
PathLockFile = ".lock"
PathTranscriptMerged = "transcripts/merged.json"
PathTranscriptProcessed = "transcripts/processed.json"
PathTranscriptNormalized = "transcripts/normalized.json"
PathTranscriptTrimmed = "transcripts/trimmed.json"
S3CampaignsSegment = "campaigns"
S3SessionsSegment = "sessions"
S3RunsSegment = "runs"
S3CurrentSegment = "current"
S3ManifestFile = "manifest.json"
S3RunIDFile = "run_id.txt"
)
// DefaultArchivePromoteArtifacts defines the default archive promotion rules.
// Callers should copy this slice before mutating.
var DefaultArchivePromoteArtifacts = []ArchivePromotionRule{
{From: PathTranscriptTrimmed, To: PathTranscriptTrimmed},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md"},
}
// DefaultPipelineConfigSearchPaths defines the default search order for
// pipeline.yml when callers do not provide an explicit path.
//
@@ -16,3 +85,14 @@ var DefaultPipelineConfigSearchPaths = []string{
DefaultPipelineConfigPathUsrLocal,
DefaultPipelineConfigPathEtc,
}
// DefaultSessionConfigSearchPaths defines the default search order for
// session.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultSessionConfigSearchPaths = []string{
DefaultSessionConfigPathLocal,
DefaultSessionConfigPathUsrLocal,
DefaultSessionConfigPathEtc,
}

View File

@@ -5,6 +5,8 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
@@ -21,21 +23,56 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
// LoadSession loads session configuration from a YAML file with strict field checking.
func LoadSession(path string) (*SessionConfig, error) {
var cfg SessionConfig
if err := decodeStrictYAML("session", path, &cfg); err != nil {
return LoadSessionWithOptions(path, SessionLoadOptions{})
}
// SessionLoadOptions configures session template rendering behavior.
type SessionLoadOptions struct {
SessionID string
}
// LoadSessionWithOptions loads session configuration from a YAML file with
// strict field checking after template rendering.
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
sessionBytes, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
}
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
if err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
var cfg SessionConfig
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
path,
strings.TrimSpace(opts.SessionID),
strings.TrimSpace(cfg.SessionID),
)
}
return &cfg, nil
}
// Load loads and resolves combined pipeline and session configuration.
func Load(pipelinePath, sessionPath string) (*Config, error) {
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
}
// LoadWithSessionOptions loads and resolves combined pipeline and session
// configuration with session template options.
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipeline(pipelinePath)
if err != nil {
return nil, err
}
sessionCfg, err := LoadSession(sessionPath)
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
if err != nil {
return nil, err
}
@@ -55,7 +92,11 @@ func decodeStrictYAML(kind, path string, out any) error {
}
defer f.Close()
dec := yaml.NewDecoder(f)
return decodeStrictYAMLFromReader(kind, path, f, out)
}
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
dec := yaml.NewDecoder(r)
dec.KnownFields(true)
if err := dec.Decode(out); err != nil {
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
@@ -69,6 +110,36 @@ func decodeStrictYAML(kind, path string, out any) error {
return nil
}
var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
sessionID := strings.TrimSpace(opts.SessionID)
rendered := content
if sessionID != "" {
rendered = strings.ReplaceAll(rendered, "{{session_id}}", sessionID)
rendered = strings.ReplaceAll(rendered, "{{ session_id }}", sessionID)
}
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
if len(unresolved) > 0 {
vars := make([]string, 0, len(unresolved))
for _, m := range unresolved {
if len(m) > 1 {
vars = append(vars, m[1])
}
}
if len(vars) > 0 {
return "", fmt.Errorf(
"session file template rendering failed: unresolved template variable(s): %s; pass --session-id when using {{ session_id }}",
strings.Join(vars, ", "),
)
}
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
}
return rendered, nil
}
func shortName(path, fallback string) string {
base := filepath.Base(path)
if base == "." || base == string(filepath.Separator) {
@@ -81,6 +152,9 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyArchiveDefaults(&cfg.Archive)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
@@ -92,24 +166,75 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
applyScriptoriumDefaults(cfg.Scriptorium)
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
if cfg.S3 == nil {
cfg.S3 = &StorageS3Config{}
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyArchiveDefaults(cfg **ArchiveConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &ArchiveConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).PromoteArtifacts) == 0 {
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
}
for i := range (*cfg).PromoteArtifacts {
if (*cfg).PromoteArtifacts[i].Required == nil {
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = "en"
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = "30m"
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = "2s"
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(2)
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(3)
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
@@ -122,17 +247,20 @@ func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "10m"
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = "seriatim-intermediate"
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(3.0)
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
@@ -140,34 +268,14 @@ func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "3h"
}
if cfg.Modules == nil {
cfg.Modules = []string{
"glossary",
"homophones",
"glossary",
"spoken_word",
"grammar",
"homophones",
"glossary",
}
}
if cfg.BaseURL == "" {
cfg.BaseURL = "https://openrouter.ai/api/v1"
}
if cfg.Model == "" {
cfg.Model = "openrouter/google/gemma-4-31b-it"
}
if cfg.LLMConcurrency == nil {
cfg.LLMConcurrency = intPtr(1)
}
if cfg.ValidationLLMConcurrency == nil {
cfg.ValidationLLMConcurrency = intPtr(1)
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
@@ -175,8 +283,11 @@ func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "10m"
cfg.Timeout = DefaultScriptoriumTimeout
}
}
@@ -185,10 +296,10 @@ func applyTrimDefaults(cfg *TrimConfig) {
return
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = "10m"
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(false)
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
@@ -197,13 +308,13 @@ func applyNormalizeDefaults(cfg *NormalizeConfig) {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = defaultNormalizeOutputSchema
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = defaultNormalizeOutputPath
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}

View File

@@ -37,6 +37,26 @@ inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
checkDefault: true,
},
{
name: "seriatim and audita sections can be omitted",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 15s
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
checkDefault: true,
},
@@ -296,7 +316,7 @@ inputs:
wantLoadErr: "strict decode failed",
},
{
name: "missing seriatim binary fails",
name: "missing seriatim binary uses default",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -311,7 +331,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.seriatim.binary is required",
},
{
name: "invalid seriatim timeout fails",
@@ -412,7 +431,7 @@ inputs:
wantLoadErr: "strict decode failed",
},
{
name: "missing audita binary fails",
name: "missing audita binary uses default",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -429,7 +448,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.binary is required",
},
{
name: "invalid audita timeout fails",
@@ -453,7 +471,7 @@ inputs:
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.timeout must be a valid duration",
},
{
name: "empty audita modules fails",
name: "empty audita modules is valid override",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -471,7 +489,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.modules must include at least one module",
},
{
name: "empty audita module item fails",
@@ -541,7 +558,7 @@ inputs:
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.base_url must be a valid URL",
},
{
name: "invalid audita llm_concurrency fails",
name: "legacy audita llm_concurrency field fails strict decode",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -550,7 +567,7 @@ seriatim:
binary: seriatim
audita:
binary: audita
llm_concurrency: 0
llm_concurrency: 1
`,
sessionYAML: `session_id: 2026-05-03
inputs:
@@ -559,7 +576,49 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.llm_concurrency must be > 0",
wantLoadErr: "strict decode failed",
},
{
name: "invalid audita total_llm_concurrency fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
total_llm_concurrency: 0
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.total_llm_concurrency must be > 0",
},
{
name: "invalid audita proposal_llm_concurrency fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
proposal_llm_concurrency: 0
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.proposal_llm_concurrency must be > 0",
},
{
name: "invalid audita validation_llm_concurrency fails",
@@ -582,6 +641,48 @@ inputs:
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.validation_llm_concurrency must be > 0",
},
{
name: "invalid audita output_schema fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
output_schema: bad
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.output_schema must be one of: bare-segments, audita-v1",
},
{
name: "invalid audita work_dir_retention fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
work_dir_retention: sometimes
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.work_dir_retention must be one of: always, auto, never",
},
}
for _, tt := range tests {
@@ -630,6 +731,9 @@ inputs:
if cfg.Pipeline.Seriatim.Timeout != "10m" {
t.Fatalf("seriatim.timeout = %q, want %q", cfg.Pipeline.Seriatim.Timeout, "10m")
}
if cfg.Pipeline.Seriatim.Binary != "seriatim" {
t.Fatalf("seriatim.binary = %q, want %q", cfg.Pipeline.Seriatim.Binary, "seriatim")
}
if cfg.Pipeline.Seriatim.OutputSchema != "seriatim-intermediate" {
t.Fatalf("seriatim.output_schema = %q, want %q", cfg.Pipeline.Seriatim.OutputSchema, "seriatim-intermediate")
}
@@ -642,26 +746,32 @@ inputs:
if cfg.Pipeline.Audita.Timeout != "3h" {
t.Fatalf("audita.timeout = %q, want %q", cfg.Pipeline.Audita.Timeout, "3h")
}
if cfg.Pipeline.Audita.Binary != "audita" {
t.Fatalf("audita.binary = %q, want %q", cfg.Pipeline.Audita.Binary, "audita")
}
if cfg.Pipeline.Audita.LLMAPIKeyEnv != "" {
t.Fatalf("audita.llm_api_key_env = %q, want empty by default", cfg.Pipeline.Audita.LLMAPIKeyEnv)
}
if got := strings.Join(cfg.Pipeline.Audita.Modules, ","); got != "glossary,homophones,glossary,spoken_word,grammar,homophones,glossary" {
t.Fatalf("audita.modules = %q, want default sequence", got)
if cfg.Pipeline.Audita.Modules != nil {
t.Fatalf("audita.modules = %#v, want nil default (optional override)", cfg.Pipeline.Audita.Modules)
}
if cfg.Pipeline.Audita.BaseURL != "https://openrouter.ai/api/v1" {
t.Fatalf("audita.base_url = %q, want %q", cfg.Pipeline.Audita.BaseURL, "https://openrouter.ai/api/v1")
if cfg.Pipeline.Audita.BaseURL != "" {
t.Fatalf("audita.base_url = %q, want empty default", cfg.Pipeline.Audita.BaseURL)
}
if cfg.Pipeline.Audita.Model != "openrouter/google/gemma-4-31b-it" {
t.Fatalf("audita.model = %q, want %q", cfg.Pipeline.Audita.Model, "openrouter/google/gemma-4-31b-it")
}
if cfg.Pipeline.Audita.LLMConcurrency == nil || *cfg.Pipeline.Audita.LLMConcurrency != 1 {
t.Fatalf("audita.llm_concurrency = %v, want 1", cfg.Pipeline.Audita.LLMConcurrency)
if cfg.Pipeline.Audita.Model != "" {
t.Fatalf("audita.model = %q, want empty default", cfg.Pipeline.Audita.Model)
}
if cfg.Pipeline.Audita.ValidationModel != "" {
t.Fatalf("audita.validation_model = %q, want empty default", cfg.Pipeline.Audita.ValidationModel)
}
if cfg.Pipeline.Audita.ValidationLLMConcurrency == nil || *cfg.Pipeline.Audita.ValidationLLMConcurrency != 1 {
t.Fatalf("audita.validation_llm_concurrency = %v, want 1", cfg.Pipeline.Audita.ValidationLLMConcurrency)
if cfg.Pipeline.Audita.TotalLLMConcurrency != nil {
t.Fatalf("audita.total_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.TotalLLMConcurrency)
}
if cfg.Pipeline.Audita.ProposalLLMConcurrency != nil {
t.Fatalf("audita.proposal_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ProposalLLMConcurrency)
}
if cfg.Pipeline.Audita.ValidationLLMConcurrency != nil {
t.Fatalf("audita.validation_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ValidationLLMConcurrency)
}
if cfg.Pipeline.Audita.Report == nil || *cfg.Pipeline.Audita.Report != true {
t.Fatalf("audita.report = %v, want true", cfg.Pipeline.Audita.Report)
@@ -725,7 +835,8 @@ func TestValidateMissingAudioSource(t *testing.T) {
Modules: []string{"glossary", "homophones"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: intPtr(1),
TotalLLMConcurrency: intPtr(1),
ProposalLLMConcurrency: intPtr(1),
ValidationModel: "",
ValidationLLMConcurrency: intPtr(1),
Report: boolPtr(true),
@@ -733,6 +844,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
},
Session: &SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: SessionInputsConfig{
SpeakersFile: "speakers.yml",
AutocorrectFile: "autocorrect.yml",
@@ -745,7 +857,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
if err == nil {
t.Fatal("expected validation error, got nil")
}
if !strings.Contains(err.Error(), "audio_dir or at least one audio_files") {
if !strings.Contains(err.Error(), "audio_dir, at least one audio_files entry, or audio_s3") {
t.Fatalf("error = %q, want audio source guidance", err.Error())
}
if !strings.Contains(err.Error(), "session config") {
@@ -774,6 +886,12 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
}
pipelineYAML += "audita:\n binary: audita\n"
}
if !strings.Contains(sessionYAML, "\ncampaign:") && !strings.HasPrefix(sessionYAML, "campaign:") {
if !strings.HasSuffix(sessionYAML, "\n") {
sessionYAML += "\n"
}
sessionYAML += "campaign: sample-campaign\n"
}
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")

View File

@@ -6,11 +6,6 @@ import (
"gopkg.in/yaml.v3"
)
const (
defaultNormalizeOutputPath = "transcripts/normalized.json"
defaultNormalizeOutputSchema = "seriatim-intermediate"
)
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
// distinguish omitted vs explicitly empty values.
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {

View File

@@ -49,11 +49,19 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
wantLoadErr: "strict decode failed",
},
{
name: "missing binary fails when section present",
name: "missing binary defaults when section present",
scriptoriumYAML: `scriptorium:
timeout: 10m
`,
wantValidateErr: "pipeline.scriptorium.binary is required",
assert: func(t *testing.T, cfg *Config) {
t.Helper()
if cfg.Pipeline.Scriptorium == nil {
t.Fatal("scriptorium config should be present")
}
if cfg.Pipeline.Scriptorium.Binary != "scriptorium" {
t.Fatalf("scriptorium.binary = %q, want scriptorium", cfg.Pipeline.Scriptorium.Binary)
}
},
},
{
name: "enabled artifact missing prompt id fails",
@@ -108,6 +116,37 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
output_kind: session_recap
`,
},
{
name: "canonical artifact source is accepted",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
transcript:
source: narratio.transcript.trimmed
required: true
`,
},
{
name: "unknown artifact source fails validation",
scriptoriumYAML: `scriptorium:
binary: scriptorium
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
inputs:
transcript:
source: narratio.unknown
required: true
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source "narratio.unknown" is unsupported`,
},
{
name: "artifact render_debug override is accepted",
scriptoriumYAML: `scriptorium:

View File

@@ -0,0 +1,156 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadSessionWithOptionsRendersCompactPlaceholder(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{session_id}}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-04-04" {
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
}
}
func TestLoadSessionWithOptionsRendersSpacedPlaceholder(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-04-04" {
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
}
}
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "unresolved template variable") {
t.Fatalf("error = %q, want unresolved-variable context", err.Error())
}
if !strings.Contains(err.Error(), "session_id") {
t.Fatalf("error = %q, want session_id variable", err.Error())
}
}
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
unknown_field: true
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want strict-decode context", err.Error())
}
}
func TestLoadSessionWithOptionsConcreteSessionStillLoads(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
}
}

View File

@@ -0,0 +1,315 @@
package config
import (
"strings"
"testing"
)
func TestStorageS3DefaultsAndValidation(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3 == nil {
t.Fatal("storage.s3 should be initialized")
}
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != DefaultS3AccessKeyIDEnv {
t.Fatalf("storage.s3.access_key_id_env = %q, want %q", cfg.Pipeline.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != DefaultS3SecretAccessKeyEnv {
t.Fatalf("storage.s3.secret_access_key_env = %q, want %q", cfg.Pipeline.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv)
}
if cfg.Pipeline.Storage.S3.ForcePathStyle {
t.Fatalf("storage.s3.force_path_style = true, want false default")
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvNamesLoadAndValidate(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: CUSTOM_KEY_ID
secret_access_key_env: CUSTOM_SECRET
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != "CUSTOM_KEY_ID" {
t.Fatalf("storage.s3.access_key_id_env = %q, want CUSTOM_KEY_ID", cfg.Pipeline.Storage.S3.AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != "CUSTOM_SECRET" {
t.Fatalf("storage.s3.secret_access_key_env = %q, want CUSTOM_SECRET", cfg.Pipeline.Storage.S3.SecretKeyEnv)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "invalid access key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: "123BAD"
`,
wantErr: "pipeline.storage.s3.access_key_id_env must be a valid environment variable name",
},
{
name: "invalid secret key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
secret_access_key_env: "bad-name"
`,
wantErr: "pipeline.storage.s3.secret_access_key_env must be a valid environment variable name",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSpoolAndArchiveDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
}
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
t.Fatalf("spool.delete_audio_after_archive = true, want false")
}
if cfg.Pipeline.Workspace.CleanupAfterArchive {
t.Fatalf("workspace.cleanup_after_archive = true, want false")
}
if cfg.Pipeline.Archive == nil {
t.Fatal("archive should be initialized by defaults")
}
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
}
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
}
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 2 {
t.Fatalf("archive.promote_artifacts len = %d, want 2 defaults", len(cfg.Pipeline.Archive.PromoteArtifacts))
}
for i, item := range cfg.Pipeline.Archive.PromoteArtifacts {
if item.Required == nil || !*item.Required {
t.Fatalf("archive.promote_artifacts[%d].required = %#v, want true", i, item.Required)
}
}
}
func TestArchivePromotionPathValidation(t *testing.T) {
tests := []struct {
name string
ruleYML string
wantErr string
}{
{
name: "absolute from path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "/transcripts/trimmed.json"
to: "transcripts/trimmed.json"
`,
wantErr: "must be a relative path",
},
{
name: "traversal to path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "transcripts/trimmed.json"
to: "../trimmed.json"
`,
wantErr: "must not contain path traversal",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + "\n" + tt.ruleYML
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct {
name string
sessionYAML string
wantErr string
}{
{
name: "valid audio_s3 prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
},
{
name: "invalid audio_s3 absolute prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: /audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must be a relative path",
},
{
name: "invalid audio_s3 traversal prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: ../audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must not contain path traversal",
},
{
name: "local and s3 audio conflict",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_dir: ./audio
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "mutually exclusive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, tt.sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestStorageS3BucketRequiredWhenS3DependentFeatureEnabled(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
`
sessionYAML := `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3.bucket is required") {
t.Fatalf("Validate() error = %v, want bucket requirement", err)
}
}
func TestLocalAudioConfigStillValid(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}

View File

@@ -3,6 +3,8 @@ package config
import (
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
)
@@ -25,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
}
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
return fmt.Errorf("pipeline/session config invalid: %w", err)
}
return nil
}
@@ -36,6 +41,15 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateSecrets(cfg.Secrets); err != nil {
return err
}
if err := validateStorage(cfg.Storage); err != nil {
return err
}
if err := validateSpool(cfg.Spool); err != nil {
return err
}
if err := validateArchive(cfg.Archive); err != nil {
return err
}
if err := validateWhisperX(cfg.WhisperX); err != nil {
return err
}
@@ -64,6 +78,54 @@ func validatePipeline(cfg *PipelineConfig) error {
return nil
}
func validateStorage(cfg StorageConfig) error {
if cfg.S3 == nil {
return nil
}
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
}
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
return err
}
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
}
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
return err
}
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
return err
}
return nil
}
func validateSpool(cfg SpoolConfig) error {
return nil
}
func validateArchive(cfg *ArchiveConfig) error {
if cfg == nil {
return nil
}
for i, item := range cfg.PromoteArtifacts {
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
if strings.TrimSpace(item.From) == "" {
return fmt.Errorf("%s.from is required", prefix)
}
if strings.TrimSpace(item.To) == "" {
return fmt.Errorf("%s.to is required", prefix)
}
if err := validateRelativeSafePath(prefix+".from", item.From); err != nil {
return err
}
if err := validateRelativeSafePath(prefix+".to", item.To); err != nil {
return err
}
}
return nil
}
func validateSecrets(cfg *SecretsConfig) error {
if cfg == nil {
return nil
@@ -200,15 +262,12 @@ func validateAudita(cfg AuditaConfig) error {
if err := validateDuration("pipeline.audita.timeout", cfg.Timeout); err != nil {
return err
}
if len(cfg.Modules) == 0 {
return fmt.Errorf("pipeline.audita.modules must include at least one module")
}
for i, mod := range cfg.Modules {
m := strings.TrimSpace(mod)
if m == "" {
for i, m := range cfg.Modules {
module := strings.TrimSpace(m)
if module == "" {
return fmt.Errorf("pipeline.audita.modules[%d] must be non-empty", i)
}
switch m {
switch module {
case "glossary", "homophones", "spoken_word", "grammar":
default:
return fmt.Errorf("pipeline.audita.modules[%d] must be one of: glossary, homophones, spoken_word, grammar", i)
@@ -223,21 +282,31 @@ func validateAudita(cfg AuditaConfig) error {
return fmt.Errorf("pipeline.audita.base_url must be a valid URL")
}
}
if strings.TrimSpace(cfg.Model) == "" {
return fmt.Errorf("pipeline.audita.model is required")
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.total_llm_concurrency must be > 0")
}
if cfg.LLMConcurrency == nil {
return fmt.Errorf("pipeline.audita.llm_concurrency must be set (defaults should populate this)")
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.proposal_llm_concurrency must be > 0")
}
if *cfg.LLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.llm_concurrency must be > 0")
}
if cfg.ValidationLLMConcurrency == nil {
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be set (defaults should populate this)")
}
if *cfg.ValidationLLMConcurrency <= 0 {
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be > 0")
}
if strings.TrimSpace(cfg.TranscriptDescription) == "" && cfg.TranscriptDescription != "" {
return fmt.Errorf("pipeline.audita.transcript_description must be non-empty when provided")
}
if strings.TrimSpace(cfg.ConfigPath) == "" && cfg.ConfigPath != "" {
return fmt.Errorf("pipeline.audita.config_path must be non-empty when provided")
}
switch strings.TrimSpace(cfg.OutputSchema) {
case "", "bare-segments", "audita-v1":
default:
return fmt.Errorf("pipeline.audita.output_schema must be one of: bare-segments, audita-v1")
}
switch strings.TrimSpace(cfg.WorkDirRetention) {
case "", "always", "auto", "never":
default:
return fmt.Errorf("pipeline.audita.work_dir_retention must be one of: always, auto, never")
}
return nil
}
@@ -274,9 +343,13 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
if trimmedInputName == "" {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
}
if strings.TrimSpace(inputCfg.Source) == "" {
source := strings.TrimSpace(inputCfg.Source)
if source == "" {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
}
if !isSupportedScriptoriumInputSource(source) {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", artifactName, inputName, inputCfg.Source)
}
}
for varName, varValue := range artifactCfg.Vars {
if strings.TrimSpace(varName) == "" {
@@ -297,6 +370,9 @@ func validateSession(cfg *SessionConfig) error {
if strings.TrimSpace(cfg.SessionID) == "" {
return fmt.Errorf("session.session_id is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("session.campaign is required")
}
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
return fmt.Errorf("session.inputs.speakers_file is required")
@@ -310,13 +386,118 @@ func validateSession(cfg *SessionConfig) error {
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
if !hasAudioDir && !hasAudioFiles {
return fmt.Errorf("session.inputs requires audio_dir or at least one audio_files entry")
hasAudioS3 := cfg.Inputs.AudioS3 != nil
if hasAudioS3 {
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
}
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
return err
}
}
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
}
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
}
return nil
}
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
if pipeline == nil || session == nil {
return nil
}
if pipeline.Storage.S3 == nil {
return nil
}
audioS3Enabled := session.Inputs.AudioS3 != nil
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
}
return nil
}
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
if pipeline == nil || pipeline.Archive == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
return false
}
enabled := true
if pipeline.Archive.Enabled != nil {
enabled = *pipeline.Archive.Enabled
}
upload := true
if pipeline.Archive.UploadRun != nil {
upload = *pipeline.Archive.UploadRun
}
return enabled && upload
}
func isSupportedScriptoriumInputSource(source string) bool {
switch strings.TrimSpace(source) {
case "previous_session_artifact":
return true
case "processed_transcript":
return true
case "normalized_transcript":
return true
case "trimmed_transcript":
return true
case "narratio.transcript.merged":
return true
case "narratio.transcript.polished":
return true
case "narratio.transcript.full":
return true
case "narratio.transcript.trimmed":
return true
case "narratio.bounds.session":
return true
case "narratio.artifact.session_recap":
return true
default:
return false
}
}
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
func validateEnvVarNameField(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if !envVarNameRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a valid environment variable name", fieldName)
}
return nil
}
func validateRelativeSafePath(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a relative path", fieldName)
}
normalized := strings.ReplaceAll(trimmed, "\\", "/")
for _, segment := range strings.Split(normalized, "/") {
if segment == ".." {
return fmt.Errorf("%s must not contain path traversal", fieldName)
}
}
return nil
}
func validateDuration(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {

View File

@@ -14,17 +14,25 @@ type ErrorRecord struct {
// InputRecord captures one resolved input and optional checksum.
type InputRecord struct {
Kind string `json:"kind"`
Path string `json:"path"`
Checksum string `json:"checksum,omitempty"`
Kind string `json:"kind"`
Path string `json:"path"`
Checksum string `json:"checksum,omitempty"`
Source string `json:"source,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3Key string `json:"s3_key,omitempty"`
S3Size int64 `json:"s3_size,omitempty"`
S3ETag string `json:"s3_etag,omitempty"`
SpoolPath string `json:"spool_path,omitempty"`
}
// ArtifactRecord captures one produced artifact and optional remote metadata.
type ArtifactRecord struct {
Kind string `json:"kind"`
LocalPath string `json:"local_path"`
RemoteKey string `json:"remote_key,omitempty"`
Checksum string `json:"checksum,omitempty"`
// ProducerRunID identifies the run that produced this durable artifact.
ProducerRunID string `json:"producer_run_id,omitempty"`
RemoteKey string `json:"remote_key,omitempty"`
Checksum string `json:"checksum,omitempty"`
}
// StageRecord tracks lifecycle and provenance for one pipeline stage.
@@ -45,6 +53,13 @@ type StageRecord struct {
// Manifest is the durable run-state record for a session execution.
type Manifest struct {
SessionID string `json:"session_id"`
Campaign string `json:"campaign,omitempty"`
RunID string `json:"run_id,omitempty"`
LocalWorkDir string `json:"local_workdir,omitempty"`
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
PipelineVersion string `json:"pipeline_version,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
@@ -107,6 +122,15 @@ func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
m.UpdatedAt = at
}
// MarkStageStale marks a stage as stale so it is not skipped as idempotently complete.
func (m *Manifest) MarkStageStale(name string, at time.Time, reason string) {
s := m.ensureStage(name, at)
s.Status = StatusStale
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "stale", At: timePtr(at)}
s.UpdatedAt = at
m.UpdatedAt = at
}
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
if m.Stages == nil {
m.Stages = map[string]*StageRecord{}

View File

@@ -0,0 +1,164 @@
package manifest
import (
"strings"
"time"
)
type RunManifestStatus string
const (
RunManifestStatusRunning RunManifestStatus = "running"
RunManifestStatusSucceeded RunManifestStatus = "succeeded"
RunManifestStatusFailed RunManifestStatus = "failed"
)
type RunStageAction string
const (
RunStageActionRun RunStageAction = "run"
RunStageActionSkip RunStageAction = "skip"
)
// RunStageRecord tracks lifecycle and provenance for one stage within a single invocation.
type RunStageRecord struct {
Name string `json:"name"`
Action RunStageAction `json:"action"`
Status StageStatus `json:"status"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
Outputs []ArtifactRecord `json:"outputs,omitempty"`
Logs []string `json:"logs,omitempty"`
GeneratedConfigs []string `json:"generated_configs,omitempty"`
Error *ErrorRecord `json:"error,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
// RunManifest is the invocation-scoped execution record under runs/{run_id}/manifest.json.
type RunManifest struct {
SessionID string `json:"session_id"`
Campaign string `json:"campaign,omitempty"`
RunID string `json:"run_id"`
Force bool `json:"force"`
RequestedStages []string `json:"requested_stages,omitempty"`
SessionManifestPath string `json:"session_manifest_path,omitempty"`
LocalWorkDir string `json:"local_workdir,omitempty"`
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
Status RunManifestStatus `json:"status"`
LastError *ErrorRecord `json:"last_error,omitempty"`
Stages map[string]*RunStageRecord `json:"stages"`
Metadata map[string]any `json:"metadata,omitempty"`
}
// NewRun constructs a new run manifest with deterministic timestamps.
func NewRun(sessionID, campaign, runID string, force bool, requestedStages []string, now time.Time) *RunManifest {
return &RunManifest{
SessionID: strings.TrimSpace(sessionID),
Campaign: strings.TrimSpace(campaign),
RunID: strings.TrimSpace(runID),
Force: force,
RequestedStages: append([]string(nil), requestedStages...),
CreatedAt: now,
UpdatedAt: now,
StartedAt: timePtr(now),
Status: RunManifestStatusRunning,
Stages: map[string]*RunStageRecord{},
}
}
func (m *RunManifest) SetStageAction(name string, action RunStageAction, at time.Time) {
s := m.ensureStage(name, at)
s.Action = action
s.UpdatedAt = at
m.UpdatedAt = at
}
func (m *RunManifest) MarkStageRunning(name string, at time.Time) {
s := m.ensureStage(name, at)
s.Status = StatusRunning
s.StartedAt = timePtr(at)
s.CompletedAt = nil
s.Error = nil
s.UpdatedAt = at
m.UpdatedAt = at
}
func (m *RunManifest) MarkStageSucceeded(name string, at time.Time, outputs []ArtifactRecord) {
s := m.ensureStage(name, at)
s.Status = StatusSucceeded
s.CompletedAt = timePtr(at)
s.Error = nil
s.Outputs = append([]ArtifactRecord(nil), outputs...)
s.UpdatedAt = at
m.UpdatedAt = at
}
func (m *RunManifest) MarkStageFailed(name string, at time.Time, message string) {
s := m.ensureStage(name, at)
s.Status = StatusFailed
s.CompletedAt = timePtr(at)
s.Error = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
s.UpdatedAt = at
m.LastError = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
m.UpdatedAt = at
m.Status = RunManifestStatusFailed
m.CompletedAt = timePtr(at)
}
func (m *RunManifest) MarkStageSkipped(name string, at time.Time, reason string) {
s := m.ensureStage(name, at)
s.Status = StatusSkipped
s.CompletedAt = timePtr(at)
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
s.UpdatedAt = at
m.UpdatedAt = at
}
func (m *RunManifest) MarkSucceeded(at time.Time) {
m.Status = RunManifestStatusSucceeded
m.CompletedAt = timePtr(at)
m.UpdatedAt = at
}
func (m *RunManifest) MarkFailed(at time.Time, message string) {
m.Status = RunManifestStatusFailed
m.CompletedAt = timePtr(at)
m.LastError = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
m.UpdatedAt = at
}
func (m *RunManifest) ensureStage(name string, at time.Time) *RunStageRecord {
if m.Stages == nil {
m.Stages = map[string]*RunStageRecord{}
}
stageName := strings.TrimSpace(name)
s, ok := m.Stages[stageName]
if !ok || s == nil {
s = &RunStageRecord{
Name: stageName,
Action: RunStageActionRun,
Status: StatusPending,
CreatedAt: at,
UpdatedAt: at,
}
m.Stages[stageName] = s
}
if s.Name == "" {
s.Name = stageName
}
if s.CreatedAt.IsZero() {
s.CreatedAt = at
}
return s
}

View File

@@ -0,0 +1,50 @@
package manifest
import (
"testing"
"time"
)
func TestRunManifestStageMarkHelpers(t *testing.T) {
rm := NewRun("2026-05-03", "forsaken", "20260517T000000Z-abcdef12", false, []string{"prepare"}, time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
if rm.Status != RunManifestStatusRunning {
t.Fatalf("status = %q, want %q", rm.Status, RunManifestStatusRunning)
}
runningAt := time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC)
rm.SetStageAction("prepare", RunStageActionRun, runningAt)
rm.MarkStageRunning("prepare", runningAt)
rm.MarkStageSucceeded("prepare", runningAt.Add(30*time.Second), []ArtifactRecord{
{Kind: "input", LocalPath: "inputs/session.yml"},
})
stage := rm.Stages["prepare"]
if stage == nil {
t.Fatal("prepare stage missing")
}
if stage.Action != RunStageActionRun {
t.Fatalf("action = %q, want %q", stage.Action, RunStageActionRun)
}
if stage.Status != StatusSucceeded {
t.Fatalf("status = %q, want %q", stage.Status, StatusSucceeded)
}
skippedAt := runningAt.Add(1 * time.Minute)
rm.SetStageAction("notify", RunStageActionSkip, skippedAt)
rm.MarkStageSkipped("notify", skippedAt, "already_succeeded")
skipped := rm.Stages["notify"]
if skipped == nil {
t.Fatal("notify stage missing")
}
if skipped.Action != RunStageActionSkip {
t.Fatalf("action = %q, want %q", skipped.Action, RunStageActionSkip)
}
if skipped.Status != StatusSkipped {
t.Fatalf("status = %q, want %q", skipped.Status, StatusSkipped)
}
rm.MarkSucceeded(skippedAt.Add(10 * time.Second))
if rm.Status != RunManifestStatusSucceeded {
t.Fatalf("status = %q, want %q", rm.Status, RunManifestStatusSucceeded)
}
}

View File

@@ -129,6 +129,89 @@ func (s *LocalStore) Save(ctx context.Context, path string, m *Manifest) error {
return nil
}
// CreateRun returns a new in-memory run manifest for one invocation.
func (s *LocalStore) CreateRun(
ctx context.Context,
sessionID, campaign, runID string,
force bool,
requestedStages []string,
) (*RunManifest, error) {
if err := checkContext(ctx); err != nil {
return nil, err
}
if strings.TrimSpace(sessionID) == "" {
return nil, fmt.Errorf("create run manifest: session_id is required")
}
if strings.TrimSpace(runID) == "" {
return nil, fmt.Errorf("create run manifest: run_id is required")
}
now := time.Now().UTC()
return NewRun(sessionID, campaign, runID, force, requestedStages, now), nil
}
// LoadRun reads and validates a local JSON run manifest from path.
func (s *LocalStore) LoadRun(ctx context.Context, path string) (*RunManifest, error) {
if err := checkContext(ctx); err != nil {
return nil, err
}
if strings.TrimSpace(path) == "" {
return nil, fmt.Errorf("load run manifest: path is required")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("load run manifest %q: %w", path, err)
}
var m RunManifest
if err := json.Unmarshal(data, &m); err != nil {
return nil, fmt.Errorf("decode run manifest %q: %w", path, err)
}
if err := validateLoadedRunManifest(&m); err != nil {
return nil, fmt.Errorf("run manifest %q invalid: %w", path, err)
}
normalizeRunManifest(&m)
return &m, nil
}
// SaveRun writes the run manifest to path atomically via temp file + rename.
func (s *LocalStore) SaveRun(ctx context.Context, path string, m *RunManifest) error {
if err := checkContext(ctx); err != nil {
return err
}
if strings.TrimSpace(path) == "" {
return fmt.Errorf("save run manifest: path is required")
}
if m == nil {
return fmt.Errorf("save run manifest: manifest is nil")
}
if strings.TrimSpace(m.SessionID) == "" {
return fmt.Errorf("save run manifest: session_id is required")
}
if strings.TrimSpace(m.RunID) == "" {
return fmt.Errorf("save run manifest: run_id is required")
}
if m.CreatedAt.IsZero() {
return fmt.Errorf("save run manifest: created_at is required")
}
m.UpdatedAt = time.Now().UTC()
if m.Stages == nil {
m.Stages = map[string]*RunStageRecord{}
}
data, err := json.MarshalIndent(m, "", " ")
if err != nil {
return fmt.Errorf("save run manifest: marshal: %w", err)
}
data = append(data, '\n')
return writeJSONAtomically(ctx, path, ".run-manifest.json.tmp-*", data)
}
func validateLoadedManifest(m *Manifest) error {
if m == nil {
return fmt.Errorf("manifest is nil")
@@ -161,6 +244,88 @@ func normalizeManifest(m *Manifest) {
}
}
func validateLoadedRunManifest(m *RunManifest) error {
if m == nil {
return fmt.Errorf("manifest is nil")
}
if strings.TrimSpace(m.SessionID) == "" {
return fmt.Errorf("session_id is required")
}
if strings.TrimSpace(m.RunID) == "" {
return fmt.Errorf("run_id is required")
}
if m.CreatedAt.IsZero() {
return fmt.Errorf("created_at is required")
}
if m.UpdatedAt.IsZero() {
return fmt.Errorf("updated_at is required")
}
return nil
}
func normalizeRunManifest(m *RunManifest) {
if m.Stages == nil {
m.Stages = map[string]*RunStageRecord{}
}
for name, stage := range m.Stages {
if stage == nil {
stage = &RunStageRecord{
Name: name,
Action: RunStageActionRun,
Status: StatusPending,
CreatedAt: m.CreatedAt,
UpdatedAt: m.UpdatedAt,
}
m.Stages[name] = stage
}
if stage.Name == "" {
stage.Name = name
}
}
}
func writeJSONAtomically(ctx context.Context, path, tempPattern string, data []byte) error {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create directory %q: %w", dir, err)
}
tmp, err := os.CreateTemp(dir, tempPattern)
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
tmpName := tmp.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpName)
}
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write temp file: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return fmt.Errorf("sync temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close temp file: %w", err)
}
if err := checkContext(ctx); err != nil {
return err
}
if err := os.Rename(tmpName, path); err != nil {
return fmt.Errorf("rename temp file: %w", err)
}
removeTmp = false
return nil
}
func checkContext(ctx context.Context) error {
if ctx == nil {
return nil

View File

@@ -22,6 +22,13 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
now := time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC)
m.MarkStageRunning("prepare", now)
m.MarkStageSucceeded("prepare", now.Add(2*time.Second), []ArtifactRecord{{Kind: "transcript", LocalPath: "transcripts/merged.json"}})
m.Campaign = "forsaken"
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = "/var/lib/narratio/work/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4"
m.LocalSpoolDir = "/var/spool/narratio/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4/audio"
m.S3Bucket = "my-dnd-archive"
m.S3SessionPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/"
m.S3RunPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/runs/20260515T031522Z-a1b2c3d4/"
path := filepath.Join(t.TempDir(), "manifest.json")
if err := store.Save(ctx, path, m); err != nil {
@@ -36,6 +43,12 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
if loaded.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
}
if loaded.Campaign != "forsaken" {
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
}
if loaded.RunID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("RunID = %q, want run id", loaded.RunID)
}
stage, ok := loaded.Stages["prepare"]
if !ok {
t.Fatalf("stage prepare not found")
@@ -137,3 +150,73 @@ func TestLoadRejectsInvalidManifest(t *testing.T) {
t.Fatalf("error = %q, want session_id validation", err.Error())
}
}
func TestLocalStoreCreateSaveLoadRunManifestRoundTrip(t *testing.T) {
store := &LocalStore{}
ctx := context.Background()
run, err := store.CreateRun(
ctx,
"2026-05-03",
"forsaken",
"20260517T000000Z-abcdef12",
true,
[]string{"prepare", "transcribe"},
)
if err != nil {
t.Fatalf("CreateRun() error = %v", err)
}
run.SessionManifestPath = "/var/lib/narratio/work/forsaken/2026-05-03/manifest.json"
run.MarkStageRunning("prepare", time.Date(2026, 5, 3, 12, 1, 0, 0, time.UTC))
run.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 12, 2, 0, 0, time.UTC), []ArtifactRecord{
{Kind: "input", LocalPath: "inputs/session.yml"},
})
run.MarkSucceeded(time.Date(2026, 5, 3, 12, 3, 0, 0, time.UTC))
path := filepath.Join(t.TempDir(), "run-manifest.json")
if err := store.SaveRun(ctx, path, run); err != nil {
t.Fatalf("SaveRun() error = %v", err)
}
loaded, err := store.LoadRun(ctx, path)
if err != nil {
t.Fatalf("LoadRun() error = %v", err)
}
if loaded.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
}
if loaded.Campaign != "forsaken" {
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
}
if loaded.RunID != "20260517T000000Z-abcdef12" {
t.Fatalf("RunID = %q, want %q", loaded.RunID, "20260517T000000Z-abcdef12")
}
if loaded.Status != RunManifestStatusSucceeded {
t.Fatalf("Status = %q, want %q", loaded.Status, RunManifestStatusSucceeded)
}
if loaded.Stages["prepare"] == nil || loaded.Stages["prepare"].Status != StatusSucceeded {
t.Fatalf("prepare stage = %#v, want succeeded", loaded.Stages["prepare"])
}
if loaded.Stages["prepare"].Action != RunStageActionRun {
t.Fatalf("prepare action = %q, want %q", loaded.Stages["prepare"].Action, RunStageActionRun)
}
}
func TestLoadRunRejectsInvalidManifest(t *testing.T) {
store := &LocalStore{}
ctx := context.Background()
path := filepath.Join(t.TempDir(), "run-manifest.json")
if err := os.WriteFile(path, []byte(`{"session_id":"2026-05-03"}`), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
_, err := store.LoadRun(ctx, path)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "run_id is required") {
t.Fatalf("error = %q, want run_id validation", err.Error())
}
}

View File

@@ -3,6 +3,7 @@ package stage
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -58,7 +59,11 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("analyze: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "analyze")
if err != nil {
return nil, fmt.Errorf("analyze: resolve run-stage layout: %w", err)
}
if env.Config.Pipeline.Scriptorium == nil {
return &StageResult{
Metadata: map[string]any{
@@ -83,27 +88,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
}, nil
}
processedTranscriptPath, processedSource, err := discoverProcessedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("analyze: resolve processed transcript: %w", err)
}
normalizedTranscriptPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("analyze: resolve normalized transcript: %w", err)
}
trimmedTranscriptPath, trimmedSource, err := discoverTrimmedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("analyze: resolve trimmed transcript: %w", err)
}
transcriptInputs := analyzeTranscriptInputs{
ProcessedPath: processedTranscriptPath,
ProcessedSource: processedSource,
NormalizedPath: normalizedTranscriptPath,
NormalizedSource: normalizedSource,
TrimmedPath: trimmedTranscriptPath,
TrimmedSource: trimmedSource,
}
transcriptRefs := discoverAnalyzeTranscriptRefs(m, paths)
inputPaths := map[string]string{}
omittedOptionalInputs := []string{}
@@ -111,7 +96,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
inputNames := sortedScriptoriumInputNames(artifactCfg.Inputs)
for _, inputName := range inputNames {
inputCfg := artifactCfg.Inputs[inputName]
resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, transcriptInputs, paths, sessionDir)
resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, m, paths, sessionDir)
if resolveErr != nil {
return nil, fmt.Errorf("analyze: resolve input %q: %w", inputName, resolveErr)
}
@@ -130,13 +115,22 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
return nil, fmt.Errorf("analyze: resolve vars: %w", err)
}
outputPath, err := resolveScriptoriumOutputPath(paths, artifactCfg.OutputPath)
canonicalOutputPath, err := resolveScriptoriumOutputPath(paths, artifactCfg.OutputPath)
if err != nil {
return nil, fmt.Errorf("analyze: resolve output path: %w", err)
}
outputPath, err := runLocalPathForCanonical(runLayout, paths, canonicalOutputPath)
if err != nil {
return nil, fmt.Errorf("analyze: resolve run-local output path: %w", err)
}
stdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stdout.log")
stderrLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stderr.log")
generatedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".generated.yml")
if runLayout.Enabled {
stdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".stdout.log")
stderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".stderr.log")
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium."+artifactName+".generated.yml")
}
timeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, artifactCfg.Timeout)
if err != nil {
@@ -157,20 +151,28 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
"omitted_optional_inputs": omittedOptionalInputs,
"vars": vars,
"timeout": timeout.String(),
"processed_transcript_path": processedTranscriptPath,
"processed_transcript_source": processedSource,
"normalized_transcript_path": normalizedTranscriptPath,
"normalized_transcript_source": normalizedSource,
"trimmed_transcript_path": trimmedTranscriptPath,
"trimmed_transcript_source": trimmedSource,
"processed_transcript_path": transcriptRefs.ProcessedPath,
"processed_transcript_source": transcriptRefs.ProcessedSource,
"normalized_transcript_path": transcriptRefs.NormalizedPath,
"normalized_transcript_source": transcriptRefs.NormalizedSource,
"trimmed_transcript_path": transcriptRefs.TrimmedPath,
"trimmed_transcript_source": transcriptRefs.TrimmedSource,
"render_debug_enabled": resolveRenderDebugEnabled(env.Config.Pipeline.Scriptorium.RenderDebug, artifactCfg.RenderDebug),
}
if meta["render_debug_enabled"] == true {
renderOutputPath := filepath.Join(paths.ArtifactsDir, artifactName+".render.json")
if runLayout.Enabled {
renderOutputPath = filepath.Join(runLayout.ReportsDir, artifactName+".render.json")
}
renderStdoutPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stdout.log")
renderStderrPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stderr.log")
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".render.generated.yml")
if runLayout.Enabled {
renderStdoutPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".render.stdout.log")
renderStderrPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".render.stderr.log")
renderGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium."+artifactName+".render.generated.yml")
}
renderReq := scriptorium.RenderArtifactRequest{
Binary: env.Config.Pipeline.Scriptorium.Binary,
@@ -257,17 +259,19 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
if err := requireNonEmptyFile(finalOutputPath, artifactName+" output"); err != nil {
return nil, fmt.Errorf("analyze: %w", err)
}
artifactRef := artifacts.Ref{
Kind: artifactName,
Category: "artifacts",
SessionID: sessionID,
AbsolutePath: finalOutputPath,
promotedArtifact, err := promoteRunLocalOutput(env.ArtifactStore, finalOutputPath, canonicalOutputPath, artifacts.Ref{
Kind: artifactName,
Category: "artifacts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("analyze: promote artifact output: %w", err)
}
logPaths = append(logPaths, stdoutLogPath, stderrLogPath)
generatedConfigs = append(generatedConfigs, generatedConfigPath)
meta["output_path"] = finalOutputPath
meta["run_output_path"] = finalOutputPath
meta["output_path"] = canonicalOutputPath
meta["generated_config_path"] = generatedConfigPath
meta["stdout_log_path"] = stdoutLogPath
meta["stderr_log_path"] = stderrLogPath
@@ -286,7 +290,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
}
return &StageResult{
Outputs: []artifacts.Ref{artifactRef},
Outputs: []artifacts.Ref{promotedArtifact},
Logs: logPaths,
GeneratedConfigs: generatedConfigs,
Metadata: meta,
@@ -350,40 +354,6 @@ func discoverProcessedTranscript(m *manifest.Manifest, paths artifacts.SessionPa
return "", "", nil
}
func discoverTrimmedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) {
candidates := []string{}
if m != nil && m.Stages != nil {
if sr := m.Stages["trim"]; sr != nil {
for _, out := range sr.Outputs {
if out.Kind != "transcript_trimmed" {
continue
}
p := strings.TrimSpace(out.LocalPath)
if p == "" {
continue
}
resolved := artifacts.ResolveSessionLocalPathForRead(paths, p)
candidates = append(candidates, filepath.Clean(resolved))
}
}
}
deduped := dedupeAndSortPaths(candidates)
for _, p := range deduped {
if info, err := os.Stat(p); err == nil && !info.IsDir() {
return p, "manifest.trim.outputs", nil
}
}
fallback := filepath.Join(paths.TranscriptsDir, "trimmed.json")
if info, err := os.Stat(fallback); err == nil && !info.IsDir() {
return filepath.Clean(fallback), "fallback.transcripts_dir", nil
}
if len(deduped) > 0 {
return deduped[0], "manifest.trim.outputs", nil
}
return "", "", nil
}
type analyzeTranscriptInputs struct {
ProcessedPath string
ProcessedSource string
@@ -393,38 +363,36 @@ type analyzeTranscriptInputs struct {
TrimmedSource string
}
func discoverAnalyzeTranscriptRefs(m *manifest.Manifest, paths artifacts.SessionPaths) analyzeTranscriptInputs {
processedPath, processedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptPolished)
normalizedPath, normalizedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFull)
trimmedPath, trimmedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptTrimmed)
return analyzeTranscriptInputs{
ProcessedPath: processedPath,
ProcessedSource: processedSource,
NormalizedPath: normalizedPath,
NormalizedSource: normalizedSource,
TrimmedPath: trimmedPath,
TrimmedSource: trimmedSource,
}
}
func discoverAnalyzeArtifactRef(m *manifest.Manifest, paths artifacts.SessionPaths, source string) (string, string) {
resolved, err := artifacts.ResolveSessionArtifact(paths, m, source)
if err != nil {
return "", ""
}
return resolved.Path, resolved.Provenance
}
func resolveScriptoriumInput(
inputName string,
inputCfg config.ScriptoriumInputConfig,
transcriptInputs analyzeTranscriptInputs,
m *manifest.Manifest,
paths artifacts.SessionPaths,
sessionDir string,
) (string, bool, error) {
switch strings.TrimSpace(inputCfg.Source) {
case "processed_transcript":
if strings.TrimSpace(transcriptInputs.ProcessedPath) == "" {
return "", false, nil
}
if err := validateProcessedTranscriptOutput(transcriptInputs.ProcessedPath); err != nil {
return "", false, fmt.Errorf("processed transcript %q invalid: %w", transcriptInputs.ProcessedPath, err)
}
return transcriptInputs.ProcessedPath, true, nil
case "normalized_transcript":
if strings.TrimSpace(transcriptInputs.NormalizedPath) == "" {
return "", false, fmt.Errorf("normalized transcript input is unavailable; run normalize stage first")
}
if err := validateProcessedTranscriptOutput(transcriptInputs.NormalizedPath); err != nil {
return "", false, fmt.Errorf("normalized transcript %q invalid: %w", transcriptInputs.NormalizedPath, err)
}
return transcriptInputs.NormalizedPath, true, nil
case "trimmed_transcript":
if strings.TrimSpace(transcriptInputs.TrimmedPath) == "" {
return "", false, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first")
}
if err := validateProcessedTranscriptOutput(transcriptInputs.TrimmedPath); err != nil {
return "", false, fmt.Errorf("trimmed transcript %q invalid: %w", transcriptInputs.TrimmedPath, err)
}
return transcriptInputs.TrimmedPath, true, nil
case "previous_session_artifact":
if strings.TrimSpace(inputCfg.Path) == "" {
return "", false, nil
@@ -435,7 +403,27 @@ func resolveScriptoriumInput(
}
return resolved, true, nil
default:
return "", false, fmt.Errorf("unsupported source %q", inputCfg.Source)
resolved, err := artifacts.ResolveSessionArtifact(paths, m, inputCfg.Source)
if err == nil {
return resolved.Path, true, nil
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
normalized, normalizeErr := artifacts.NormalizeSessionArtifactSource(inputCfg.Source)
if normalizeErr != nil {
return "", false, normalizeErr
}
switch normalized {
case artifacts.ArtifactTranscriptPolished:
return "", false, nil
case artifacts.ArtifactTranscriptFull:
return "", false, fmt.Errorf("normalized transcript input is unavailable; run normalize stage first")
case artifacts.ArtifactTranscriptTrimmed:
return "", false, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first")
default:
return "", false, nil
}
}
return "", false, err
}
}

View File

@@ -17,7 +17,7 @@ import (
func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
result, err := (analyzeStage{}).Run(context.Background(), env, m)
@@ -70,7 +70,7 @@ func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) {
func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = false
@@ -86,7 +86,7 @@ func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) {
func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
@@ -109,7 +109,7 @@ func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T)
func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
@@ -130,7 +130,7 @@ func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) {
func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
@@ -154,7 +154,7 @@ func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) {
func TestAnalyzeRenderFailurePreventsRun(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
fake.RenderErr = errors.New("render boom")
@@ -176,7 +176,7 @@ func TestAnalyzeRenderFailurePreventsRun(t *testing.T) {
func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
runner := &orderedScriptoriumRunner{
@@ -199,7 +199,7 @@ func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) {
func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.RenderDebug = true
@@ -220,7 +220,7 @@ func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) {
func TestAnalyzeOmitsOptionalPreviousRecapWhenUnavailable(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
@@ -256,6 +256,32 @@ func TestAnalyzeOmitsOptionalPreviousRecapWhenUnavailable(t *testing.T) {
}
}
func TestAnalyzeUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
result, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RunRequests) != 1 {
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
}
if !strings.Contains(fake.RunRequests[0].OutputPath, filepath.Join("runs", m.RunID, "analyze", "outputs")) {
t.Fatalf("run output path = %q, want run-local path", fake.RunRequests[0].OutputPath)
}
if len(result.Outputs) != 1 {
t.Fatalf("outputs len = %d, want 1", len(result.Outputs))
}
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
}
}
type orderedScriptoriumRunner struct {
Calls []string
RenderErr error
@@ -318,7 +344,7 @@ func (r *orderedScriptoriumRunner) RunArtifact(_ context.Context, req scriptoriu
func TestAnalyzeIncludesPreviousRecapWhenConfiguredAndAvailable(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
previousRecapPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "previous", "session_recap.md")
@@ -359,7 +385,7 @@ func TestAnalyzeIncludesPreviousRecapWhenConfiguredAndAvailable(t *testing.T) {
func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
@@ -390,7 +416,7 @@ func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) {
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
@@ -425,7 +451,7 @@ func TestAnalyzeFailsWhenTrimmedTranscriptMissing(t *testing.T) {
func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
@@ -447,9 +473,33 @@ func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) {
}
}
func TestAnalyzeSupportsCanonicalTrimmedTranscriptSourceWhenConfigured(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{
Source: "narratio.transcript.trimmed",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RunRequests) != 1 {
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
}
if fake.RunRequests[0].InputPaths["transcript"] != filepath.Join(paths.TranscriptsDir, "trimmed.json") {
t.Fatalf("transcript input = %q, want trimmed transcript path", fake.RunRequests[0].InputPaths["transcript"])
}
}
func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
normalizedPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeAnalyzeFile(t, normalizedPath, `{"segments":[{"id":1}]}`)
@@ -472,9 +522,39 @@ func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) {
}
}
func TestAnalyzeSupportsCanonicalNormalizedTranscriptSourceFromManifestOutput(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
fallbackPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
writeAnalyzeFile(t, fallbackPath, `{"segments":[{"id":999}]}`)
writeAnalyzeFile(t, manifestPath, `{"segments":[{"id":10}]}`)
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
{Kind: "transcript_normalized", LocalPath: manifestPath},
})
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{
Source: "narratio.transcript.full",
Required: true,
}
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
_, err := (analyzeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(fake.RunRequests) != 1 {
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
}
if fake.RunRequests[0].InputPaths["transcript"] != manifestPath {
t.Fatalf("transcript input = %q, want manifest normalized transcript path", fake.RunRequests[0].InputPaths["transcript"])
}
}
func TestAnalyzeSupportsNormalizedTranscriptSourceFromManifestOutput(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
fallbackPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
writeAnalyzeFile(t, fallbackPath, `{"segments":[{"id":999}]}`)
@@ -528,7 +608,7 @@ func TestAnalyzeFailsWhenNormalizedTranscriptMissing(t *testing.T) {
func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{not-json`)
_, err := (analyzeStage{}).Run(context.Background(), env, m)
@@ -542,7 +622,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"not_segments":[]}`)
_, err := (analyzeStage{}).Run(context.Background(), env, m)
@@ -556,7 +636,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
result, err := (analyzeStage{}).Run(context.Background(), env, m)
@@ -585,7 +665,7 @@ func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) {
func TestAnalyzeHandlesAdapterError(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
fake.RunErr = errors.New("adapter boom")
@@ -600,7 +680,7 @@ func TestAnalyzeHandlesAdapterError(t *testing.T) {
func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) {
env, m, fake := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
fake.RunResult = scriptorium.ArtifactResult{
ValidationFailed: true,
@@ -619,7 +699,7 @@ func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) {
func TestAnalyzeSkipsWhenNoEnabledScriptoriumArtifactsConfigured(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
@@ -692,7 +772,7 @@ func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeR
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}

582
internal/stage/archive.go Normal file
View File

@@ -0,0 +1,582 @@
package stage
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type archiveStage struct{}
type archiveUploadFile struct {
RelativePath string
LocalPath string
}
var archivePrerequisiteStages = []string{
"prepare",
"transcribe",
"merge",
"polish",
"normalize",
"trim",
"analyze",
}
func (archiveStage) Name() string { return "archive" }
func (archiveStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "manifest", Category: "input", RelativePath: "manifest.json"},
},
}
}
func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return nil, fmt.Errorf("archive: resolved config must include pipeline and session")
}
if archiveDisabled(env) {
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"skipped": true,
"archive_enabled": false,
"audio_upload_skipped": true,
"current_pointer_written": false,
},
}, nil
}
if archiveRunUploadDisabled(env) {
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"skipped": true,
"upload_run_enabled": false,
"audio_upload_skipped": true,
"current_pointer_written": false,
},
}, nil
}
if err := validateArchivePrerequisites(m); err != nil {
return nil, fmt.Errorf("archive: %w", err)
}
if env.ObjectStore == nil {
return nil, fmt.Errorf("archive: remote object store backend is required when archive run upload is enabled")
}
runRoot, err := resolveArchiveRunRoot(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve run root: %w", err)
}
runRootInfo, err := os.Stat(runRoot)
if err != nil {
return nil, fmt.Errorf("archive: run root %q: %w", runRoot, err)
}
if !runRootInfo.IsDir() {
return nil, fmt.Errorf("archive: run root %q is not a directory", runRoot)
}
runPrefix, err := archiveRunPrefix(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve s3 run prefix: %w", err)
}
sessionPrefix, err := archiveSessionPrefix(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve s3 session prefix: %w", err)
}
bucket := archiveBucket(env, m)
if bucket == "" {
return nil, fmt.Errorf("archive: resolve s3 bucket: bucket is required")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return nil, fmt.Errorf("archive: run id is required")
}
manifestSource, err := resolveArchiveRunManifestSource(runRoot)
if err != nil {
return nil, fmt.Errorf("archive: resolve run manifest source: %w", err)
}
runFiles, err := collectArchiveRunFiles(runRoot, manifestSource)
if err != nil {
return nil, fmt.Errorf("archive: collect run files: %w", err)
}
sessionRoot, err := resolveArchiveSessionRoot(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve session root for promotions: %w", err)
}
promotions, err := resolveArchivePromotions(sessionRoot, env.Config.Pipeline.Archive.PromoteArtifacts)
if err != nil {
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
}
runUploaded := make([]string, 0, len(runFiles))
for _, file := range runFiles {
key := artifacts.S3RunRelativeDestinationKey(runPrefix, file.RelativePath)
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
return nil, fmt.Errorf("archive: upload run file %q to %q: %w", file.RelativePath, key, err)
}
runUploaded = append(runUploaded, file.RelativePath)
}
promotedUploaded := make([]string, 0, len(promotions))
skippedOptional := make([]string, 0)
for _, promotion := range promotions {
if !promotion.Exists {
if promotion.Required {
return nil, fmt.Errorf("archive: required promotion source missing: %q", promotion.From)
}
skippedOptional = append(skippedOptional, promotion.To)
continue
}
key := artifacts.S3PromotedArtifactKey(sessionPrefix, promotion.To)
if _, err := env.ObjectStore.Upload(ctx, promotion.LocalPath, key, storage.UploadOptions{}); err != nil {
return nil, fmt.Errorf("archive: upload promoted output %q to %q: %w", promotion.From, key, err)
}
promotedUploaded = append(promotedUploaded, promotion.To)
}
currentManifestKey := artifacts.S3CurrentManifestKey(sessionPrefix)
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
bucket,
runPrefix,
sessionPrefix,
runUploaded,
promotedUploaded,
skippedOptional,
currentManifestKey,
))
if err != nil {
return nil, fmt.Errorf("archive: build current manifest snapshot: %w", err)
}
defer func() { _ = os.Remove(manifestTempPath) }()
if _, err := env.ObjectStore.Upload(ctx, manifestTempPath, currentManifestKey, storage.UploadOptions{
ContentType: "application/json",
}); err != nil {
return nil, fmt.Errorf("archive: upload current manifest to %q: %w", currentManifestKey, err)
}
currentRunPointerKey := artifacts.S3CurrentRunPointerKey(sessionPrefix)
runIDTempPath, err := writeCurrentRunIDPointer(runID)
if err != nil {
return nil, fmt.Errorf("archive: build current run id pointer: %w", err)
}
defer func() { _ = os.Remove(runIDTempPath) }()
if _, err := env.ObjectStore.Upload(ctx, runIDTempPath, currentRunPointerKey, storage.UploadOptions{
ContentType: "text/plain; charset=utf-8",
}); err != nil {
return nil, fmt.Errorf("archive: upload current run pointer to %q: %w", currentRunPointerKey, err)
}
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"uploaded": true,
"s3_bucket": bucket,
"s3_run_prefix": runPrefix,
"run_files_uploaded": len(runUploaded),
"run_uploaded_paths": runUploaded,
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": promotedUploaded,
"skipped_optional_promotions": skippedOptional,
"current_manifest_key": currentManifestKey,
"current_run_id_key": currentRunPointerKey,
"current_pointer_written": true,
"audio_upload_skipped": true,
},
}, nil
}
type archivePromotion struct {
From string
To string
Required bool
LocalPath string
Exists bool
}
func archiveDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
return true
}
return cfg.Enabled != nil && !*cfg.Enabled
}
func archiveRunUploadDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
return true
}
return cfg.UploadRun != nil && !*cfg.UploadRun
}
func validateArchivePrerequisites(m *manifest.Manifest) error {
if m == nil {
return fmt.Errorf("manifest is required")
}
for _, stageName := range archivePrerequisiteStages {
sr := m.Stages[stageName]
if sr == nil {
return fmt.Errorf("prerequisite stage %q has not succeeded", stageName)
}
if sr.Status != manifest.StatusSucceeded {
return fmt.Errorf("prerequisite stage %q status is %q (want %q)", stageName, sr.Status, manifest.StatusSucceeded)
}
}
return nil
}
func resolveArchiveRunRoot(env *Env, m *manifest.Manifest) (string, error) {
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" && m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" && m != nil {
campaign = strings.TrimSpace(m.Campaign)
}
runID := ""
if m != nil {
runID = strings.TrimSpace(m.RunID)
}
if sessionID == "" || campaign == "" {
return "", fmt.Errorf("campaign and session id are required")
}
if runID == "" {
return "", fmt.Errorf("run id is required")
}
canonical := filepath.Clean(artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID))
canonicalExists, err := directoryExists(canonical)
if err != nil {
return "", fmt.Errorf("check canonical run root %q: %w", canonical, err)
}
if !canonicalExists {
return "", fmt.Errorf("run root not found for campaign %q session %q run %q at canonical path %q", campaign, sessionID, runID, canonical)
}
return canonical, nil
}
func resolveArchiveSessionRoot(env *Env, m *manifest.Manifest) (string, error) {
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" && m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" && m != nil {
campaign = strings.TrimSpace(m.Campaign)
}
if sessionID == "" {
return "", fmt.Errorf("session id is required")
}
if campaign == "" {
return "", fmt.Errorf("campaign is required")
}
return filepath.Clean(artifacts.SessionWorkDirForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID)), nil
}
func archiveRunPrefix(env *Env, m *manifest.Manifest) (string, error) {
runPrefix := strings.TrimSpace(m.S3RunPrefix)
if runPrefix != "" {
return runPrefix, nil
}
sessionPrefix, err := archiveSessionPrefix(env, m)
if err != nil {
return "", err
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return "", fmt.Errorf("run id is required")
}
return artifacts.S3RunPrefix(sessionPrefix, runID), nil
}
func archiveSessionPrefix(env *Env, m *manifest.Manifest) (string, error) {
if m != nil && strings.TrimSpace(m.S3SessionPrefix) != "" {
return strings.TrimSpace(m.S3SessionPrefix), nil
}
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
campaign = strings.TrimSpace(m.Campaign)
}
if env.Config.Pipeline.Storage.S3 == nil {
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
if strings.TrimSpace(sessionPrefix) == "" {
return "", fmt.Errorf("session prefix is required")
}
return sessionPrefix, nil
}
func archiveBucket(env *Env, m *manifest.Manifest) string {
if m != nil && strings.TrimSpace(m.S3Bucket) != "" {
return strings.TrimSpace(m.S3Bucket)
}
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Pipeline.Storage.S3 == nil {
return ""
}
return strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
}
func resolveArchivePromotions(sessionRoot string, rules []config.ArchivePromotionRule) ([]archivePromotion, error) {
sessionRoot = filepath.Clean(strings.TrimSpace(sessionRoot))
if sessionRoot == "" {
return nil, fmt.Errorf("session root is required")
}
out := make([]archivePromotion, 0, len(rules))
for _, rule := range rules {
from := strings.TrimSpace(rule.From)
to := strings.TrimSpace(rule.To)
required := rule.Required == nil || *rule.Required
resolvedPath, err := resolveWorkDirRelativePath(sessionRoot, from)
if err != nil {
return nil, fmt.Errorf("promotion from %q: %w", from, err)
}
info, err := os.Stat(resolvedPath)
exists := err == nil && !info.IsDir()
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("promotion source %q: %w", from, err)
}
localPath := resolvedPath
out = append(out, archivePromotion{
From: from,
To: to,
Required: required,
LocalPath: localPath,
Exists: exists,
})
}
return out, nil
}
func resolveWorkDirRelativePath(workDir, rel string) (string, error) {
rel = filepath.Clean(filepath.FromSlash(strings.TrimSpace(rel)))
if rel == "." || rel == "" {
return "", fmt.Errorf("relative path is required")
}
full := filepath.Join(workDir, rel)
cleanedWork := filepath.Clean(workDir)
cleanedFull := filepath.Clean(full)
relative, err := filepath.Rel(cleanedWork, cleanedFull)
if err != nil {
return "", fmt.Errorf("compute relative path: %w", err)
}
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path escapes workdir")
}
return cleanedFull, nil
}
func collectArchiveRunFiles(runRoot, manifestPath string) ([]archiveUploadFile, error) {
files := make([]archiveUploadFile, 0, 64)
err := filepath.WalkDir(runRoot, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
if path == runRoot {
return nil
}
relDir, err := filepath.Rel(runRoot, path)
if err != nil {
return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err)
}
relDir = filepath.ToSlash(relDir)
// Preserve existing behavior: audio is not uploaded in archive run record.
if relDir == "audio" || strings.HasPrefix(relDir, "audio/") {
return filepath.SkipDir
}
return nil
}
rel, err := filepath.Rel(runRoot, path)
if err != nil {
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
}
rel = filepath.ToSlash(rel)
files = append(files, archiveUploadFile{
RelativePath: rel,
LocalPath: path,
})
return nil
})
if err != nil {
return nil, fmt.Errorf("walk %q: %w", runRoot, err)
}
manifestInfo, err := os.Stat(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("manifest.json not found (checked path %q)", manifestPath)
}
return nil, fmt.Errorf("stat %q: %w", manifestPath, err)
}
if manifestInfo.IsDir() {
return nil, fmt.Errorf("manifest path %q is a directory", manifestPath)
}
files = append(files, archiveUploadFile{
RelativePath: "manifest.json",
LocalPath: manifestPath,
})
seen := map[string]archiveUploadFile{}
for _, file := range files {
seen[file.RelativePath] = file
}
files = files[:0]
for _, file := range seen {
files = append(files, file)
}
sort.Slice(files, func(i, j int) bool {
return files[i].RelativePath < files[j].RelativePath
})
return files, nil
}
func resolveArchiveRunManifestSource(runRoot string) (string, error) {
path := filepath.Join(filepath.Clean(runRoot), "manifest.json")
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return "", fmt.Errorf("manifest.json not found in run root %q", runRoot)
}
return "", fmt.Errorf("stat %q: %w", path, err)
}
if info.IsDir() {
return "", fmt.Errorf("manifest path %q is a directory", path)
}
return path, nil
}
func directoryExists(path string) (bool, error) {
info, err := os.Stat(path)
if err == nil {
return info.IsDir(), nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
func writeCurrentManifestSnapshot(m *manifest.Manifest, archiveMetadata map[string]any) (string, error) {
if m == nil {
return "", fmt.Errorf("manifest is required")
}
clone := *m
clone.Stages = make(map[string]*manifest.StageRecord, len(m.Stages))
for name, sr := range m.Stages {
if sr == nil {
continue
}
stageCopy := *sr
if sr.Outputs != nil {
stageCopy.Outputs = append([]manifest.ArtifactRecord(nil), sr.Outputs...)
}
if sr.Logs != nil {
stageCopy.Logs = append([]string(nil), sr.Logs...)
}
if sr.GeneratedConfigs != nil {
stageCopy.GeneratedConfigs = append([]string(nil), sr.GeneratedConfigs...)
}
if sr.Metadata != nil {
metaCopy := make(map[string]any, len(sr.Metadata))
for k, v := range sr.Metadata {
metaCopy[k] = v
}
stageCopy.Metadata = metaCopy
}
clone.Stages[name] = &stageCopy
}
now := time.Now().UTC()
clone.MarkStageSucceeded("archive", now, nil)
if sr := clone.Stages["archive"]; sr != nil {
sr.Metadata = archiveMetadata
}
data, err := json.MarshalIndent(&clone, "", " ")
if err != nil {
return "", fmt.Errorf("marshal manifest: %w", err)
}
data = append(data, '\n')
tmp, err := os.CreateTemp("", "narratio-current-manifest-*.json")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("write temp manifest: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp manifest: %w", err)
}
return path, nil
}
func writeCurrentRunIDPointer(runID string) (string, error) {
tmp, err := os.CreateTemp("", "narratio-current-run-id-*.txt")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if _, err := tmp.WriteString(runID + "\n"); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("write temp run id pointer: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp run id pointer: %w", err)
}
return path, nil
}
func archiveMetadataPreview(
bucket, runPrefix, sessionPrefix string,
runUploaded []string,
promotedUploaded []string,
skippedOptional []string,
currentManifestKey string,
) map[string]any {
return map[string]any{
"stage": "archive",
"uploaded": true,
"s3_bucket": bucket,
"s3_run_prefix": runPrefix,
"run_files_uploaded": len(runUploaded),
"run_uploaded_paths": append([]string(nil), runUploaded...),
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": append([]string(nil), promotedUploaded...),
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
"current_manifest_key": currentManifestKey,
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
"current_pointer_written": false,
"audio_upload_skipped": true,
}
}

View File

@@ -0,0 +1,343 @@
package stage
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestArchiveSkipsWhenDisabled(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Enabled = boolPtr(false)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["skipped"] != true {
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads when archive disabled")
}
}
func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.UploadRun = boolPtr(false)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["skipped"] != true {
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads when upload_run disabled")
}
}
func TestArchiveFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
env, m, _ := archiveFixture(t)
m.Stages["trim"].Status = manifest.StatusFailed
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), `prerequisite stage "trim"`) {
t.Fatalf("Run() error = %v, want prerequisite failure", err)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads on prerequisite failure")
}
}
func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
runPrefix := m.S3RunPrefix
sessionPrefix := m.S3SessionPrefix
wantRunUploads := []string{
"analyze/outputs/artifacts/session_recap.md",
"merge/config/seriatim.generated.yml",
"prepare/inputs/session.yml",
"prepare/outputs/audio/speaker.flac",
"logs/audita.stderr.log",
"manifest.json",
"polish/reports/audita.report.json",
"transcribe/outputs/transcripts/raw/speaker.json",
"trim/outputs/transcripts/trimmed.json",
}
for _, rel := range wantRunUploads {
key := runPrefix + rel
if _, ok := fake.Objects[key]; !ok {
t.Fatalf("missing run upload key %q", key)
}
}
trimmedKey := sessionPrefix + "transcripts/trimmed.json"
recapKey := sessionPrefix + "artifacts/session_recap.md"
if _, ok := fake.Objects[trimmedKey]; !ok {
t.Fatalf("missing promoted key %q", trimmedKey)
}
if _, ok := fake.Objects[recapKey]; !ok {
t.Fatalf("missing promoted key %q", recapKey)
}
currentManifestKey := sessionPrefix + "current/manifest.json"
currentRunIDKey := sessionPrefix + "current/run_id.txt"
if _, ok := fake.Objects[currentManifestKey]; !ok {
t.Fatalf("missing current manifest key %q", currentManifestKey)
}
if _, ok := fake.Objects[currentRunIDKey]; !ok {
t.Fatalf("missing current run pointer key %q", currentRunIDKey)
}
if got := string(fake.Objects[currentRunIDKey].Data); got != m.RunID+"\n" {
t.Fatalf("run pointer contents = %q, want %q", got, m.RunID+"\\n")
}
audioKey := runPrefix + "audio/speaker.flac"
if _, ok := fake.Objects[audioKey]; ok {
t.Fatalf("audio key %q should not be uploaded", audioKey)
}
uploads := fake.Uploads
if len(uploads) == 0 {
t.Fatal("expected uploads")
}
if uploads[len(uploads)-1].Key != currentRunIDKey {
t.Fatalf("last upload key = %q, want current run pointer key %q", uploads[len(uploads)-1].Key, currentRunIDKey)
}
if result.Metadata["current_pointer_written"] != true {
t.Fatalf("metadata = %#v, want current_pointer_written=true", result.Metadata)
}
if result.Metadata["promoted_files_uploaded"] != 2 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
}
}
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "published/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "published/recap.md", Required: boolPtr(true)},
}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
if _, ok := fake.Objects[m.S3SessionPrefix+"published/trimmed.json"]; !ok {
t.Fatalf("missing custom promoted trimmed key")
}
if _, ok := fake.Objects[m.S3SessionPrefix+"published/recap.md"]; !ok {
t.Fatalf("missing custom promoted recap key")
}
}
func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/optional.md", To: "artifacts/optional.md", Required: boolPtr(false)},
}
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
got, _ := result.Metadata["skipped_optional_promotions"].([]string)
want := []string{"artifacts/optional.md"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("skipped_optional_promotions = %#v, want %#v", got, want)
}
}
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
t.Fatalf("Run() error = %v, want required promotion missing failure", err)
}
}
func TestArchiveFailsWhenCanonicalRunRootMissing(t *testing.T) {
env, m, runRoot := archiveFixture(t)
if err := os.RemoveAll(runRoot); err != nil {
t.Fatalf("remove run root: %v", err)
}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected missing run-root error, got nil")
}
if !strings.Contains(err.Error(), "run root not found") {
t.Fatalf("error = %v, want missing run-root error", err)
}
}
func TestArchiveDoesNotWriteCurrentPointerWhenPromotionUploadFails(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
origUploadErr := fake.UploadErr
fake.UploadErr = nil
failingKey := trimmedKey
fake.Uploads = nil
originalUpload := fake.Upload
_ = originalUpload
// Use UploadErr toggle by checking call sequence in postcondition.
// First failure point is promotion upload; simulate by setting error immediately before promotion key write.
// We cannot hook FakeBackend per-key without changing public behavior; use dedicated backend wrapper instead.
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: failingKey}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "promoted output") {
t.Fatalf("Run() error = %v, want promotion upload failure", err)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
t.Fatalf("unexpected current pointer write on promotion failure")
}
fake.UploadErr = origUploadErr
}
func TestArchiveDoesNotWriteCurrentPointerWhenCurrentManifestUploadFails(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: m.S3SessionPrefix + "current/manifest.json"}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "current manifest") {
t.Fatalf("Run() error = %v, want current manifest upload failure", err)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
t.Fatalf("unexpected current pointer write when current manifest upload fails")
}
}
func TestArchiveFailsWithoutObjectStore(t *testing.T) {
env, m, _ := archiveFixture(t)
env.ObjectStore = nil
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "object store") {
t.Fatalf("Run() error = %v, want object store backend failure", err)
}
}
func archiveFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
t.Helper()
root := t.TempDir()
runID := "20260516T010203Z-1a2b3c4d"
campaign := "forsaken"
sessionID := "2026-04-19"
sessionRoot := artifacts.SessionWorkDirForCampaign(root, campaign, sessionID)
runRoot := artifacts.SessionRunRootForCampaign(root, campaign, sessionID, runID)
writeStageTestFile(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), "{}\n")
writeStageTestFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
writeStageTestFile(t, filepath.Join(runRoot, "prepare", "inputs", "session.yml"), "session_id: 2026-04-19\n")
writeStageTestFile(t, filepath.Join(runRoot, "prepare", "outputs", "audio", "speaker.flac"), "flac\n")
writeStageTestFile(t, filepath.Join(runRoot, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
writeStageTestFile(t, filepath.Join(runRoot, "trim", "outputs", "transcripts", "trimmed.json"), "{}\n")
writeStageTestFile(t, filepath.Join(runRoot, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
writeStageTestFile(t, filepath.Join(runRoot, "polish", "reports", "audita.report.json"), "{}\n")
writeStageTestFile(t, filepath.Join(runRoot, "merge", "config", "seriatim.generated.yml"), "key: value\n")
writeStageTestFile(t, filepath.Join(runRoot, "logs", "audita.stderr.log"), "stderr\n")
writeStageTestFile(t, filepath.Join(runRoot, "audio", "speaker.flac"), "flac")
writeStageTestFile(t, filepath.Join(runRoot, "manifest.json"), "{}\n")
m := manifest.New(sessionID, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC))
m.Campaign = campaign
m.RunID = runID
m.LocalWorkDir = runRoot
m.S3Bucket = "my-dnd-archive"
m.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", campaign, sessionID)
m.S3RunPrefix = artifacts.S3RunPrefix(m.S3SessionPrefix, runID)
for _, name := range archivePrerequisiteStages {
m.MarkStageSucceeded(name, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), nil)
}
env := &Env{
Config: &config.Config{
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{
S3: &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
},
},
Archive: &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
PromoteArtifacts: []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
},
},
},
Session: &config.SessionConfig{
SessionID: sessionID,
Campaign: campaign,
},
},
ObjectStore: &storage.FakeBackend{},
}
return env, m, runRoot
}
type promotionFailingStore struct {
delegate *storage.FakeBackend
failKey string
}
func (s *promotionFailingStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
return s.delegate.List(ctx, prefix)
}
func (s *promotionFailingStore) Download(ctx context.Context, key, localPath string) error {
return s.delegate.Download(ctx, key, localPath)
}
func (s *promotionFailingStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
return storage.ObjectInfo{}, errors.New("forced upload failure")
}
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *promotionFailingStore) Exists(ctx context.Context, key string) (bool, error) {
return s.delegate.Exists(ctx, key)
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -7,6 +7,7 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
@@ -56,7 +57,11 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
return nil, fmt.Errorf("merge: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "merge")
if err != nil {
return nil, fmt.Errorf("merge: resolve run-stage layout: %w", err)
}
inputs, err := discoverRawTranscripts(m, paths)
if err != nil {
@@ -80,16 +85,37 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
return nil, fmt.Errorf("merge: %w", err)
}
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
reportPath := filepath.Join(paths.ArtifactsDir, "seriatim.report.json")
canonicalMergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
mergedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalMergedPath)
if err != nil {
return nil, fmt.Errorf("merge: resolve run-local merged transcript path: %w", err)
}
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "seriatim.report.json")
reportPath := canonicalReportPath
if runLayout.Enabled {
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
if err != nil {
return nil, fmt.Errorf("merge: resolve run-local report path: %w", err)
}
}
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.stdout.log")
stderrPath := filepath.Join(paths.LogsDir, "seriatim.stderr.log")
genCfgPath := filepath.Join(paths.ConfigDir, "seriatim.generated.yml")
if runLayout.Enabled {
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.stdout.log")
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.stderr.log")
genCfgPath = filepath.Join(runLayout.ConfigDir, "seriatim.generated.yml")
}
normalizedInputs, normalizeLogs, normalizeConfigs, normalizeMeta, err := normalizeMergeInputs(ctx, env, inputs, paths, runLayout)
if err != nil {
return nil, err
}
reportEnabled := env.Config.Pipeline.Seriatim.Report != nil && *env.Config.Pipeline.Seriatim.Report
req := seriatim.MergeRequest{
GeneratedConfigPath: genCfgPath,
InputTranscriptPaths: inputs,
InputTranscriptPaths: normalizedInputs,
OutputMergedTranscriptPath: mergedPath,
ReportPath: "",
SpeakersPath: speakersPath,
@@ -124,35 +150,50 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
}
}
outputs := []artifacts.Ref{{
Kind: "transcript_merged",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: finalMergedPath,
}}
promotedMerged, err := promoteRunLocalOutput(env.ArtifactStore, finalMergedPath, canonicalMergedPath, artifacts.Ref{
Kind: "transcript_merged",
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("merge: promote merged transcript: %w", err)
}
outputs := []artifacts.Ref{promotedMerged}
if reportEnabled {
outputs = append(outputs, artifacts.Ref{
Kind: "seriatim_report",
Category: "artifacts",
SessionID: sessionID,
AbsolutePath: finalReportPath,
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
Kind: "seriatim_report",
Category: "artifacts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("merge: promote report: %w", err)
}
outputs = append(outputs, promotedReport)
}
coalesceGap := any(nil)
if env.Config.Pipeline.Seriatim.CoalesceGap != nil {
coalesceGap = *env.Config.Pipeline.Seriatim.CoalesceGap
}
reportCanonicalPath := ""
if reportEnabled {
reportCanonicalPath = canonicalReportPath
}
meta := map[string]any{
"stage": "merge",
"input_transcripts_count": len(inputs),
"input_transcripts_count": len(normalizedInputs),
"input_transcript_paths": inputs,
"normalized_inputs_count": len(normalizedInputs),
"normalized_input_paths": normalizedInputs,
"normalize_inputs": normalizeMeta,
"output_schema": env.Config.Pipeline.Seriatim.OutputSchema,
"coalesce_gap": coalesceGap,
"report_enabled": reportEnabled,
"output_path": finalMergedPath,
"report_path": finalReportPath,
"run_output_path": finalMergedPath,
"output_path": canonicalMergedPath,
"run_report_path": finalReportPath,
"report_path": reportCanonicalPath,
"timeout": env.Config.Pipeline.Seriatim.Timeout,
"binary": env.Config.Pipeline.Seriatim.Binary,
"adapter_duration_ms": res.Duration.Milliseconds(),
@@ -172,12 +213,111 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
return &StageResult{
Outputs: outputs,
Logs: []string{stdoutPath, stderrPath},
GeneratedConfigs: []string{genCfgPath},
Logs: append(normalizeLogs, stdoutPath, stderrPath),
GeneratedConfigs: append(normalizeConfigs, genCfgPath),
Metadata: meta,
}, nil
}
type normalizeMergeInputMeta struct {
InputPath string `json:"input_path"`
OutputPath string `json:"output_path"`
StdoutLogPath string `json:"stdout_log_path"`
StderrLogPath string `json:"stderr_log_path"`
GeneratedConfig string `json:"generated_config_path"`
DurationMs int64 `json:"duration_ms"`
ExitCode int `json:"exit_code"`
InvokedBinary string `json:"invoked_binary"`
OutputSchema string `json:"output_schema"`
AdapterReportPath string `json:"adapter_report_path,omitempty"`
AdapterOutputPath string `json:"adapter_output_path,omitempty"`
}
func normalizeMergeInputs(
ctx context.Context,
env *Env,
rawInputs []string,
paths artifacts.SessionPaths,
runLayout runStageLayout,
) ([]string, []string, []string, []normalizeMergeInputMeta, error) {
normalizedInputs := make([]string, 0, len(rawInputs))
logs := make([]string, 0, len(rawInputs)*2)
configs := make([]string, 0, len(rawInputs))
meta := make([]normalizeMergeInputMeta, 0, len(rawInputs))
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
if runLayout.Enabled {
normalizedDir = filepath.Join(runLayout.ScratchDir, "normalized")
}
if err := os.MkdirAll(normalizedDir, 0o755); err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: ensure normalized transcripts directory %q: %w", normalizedDir, err)
}
var timeout time.Duration
timeoutRaw := strings.TrimSpace(env.Config.Pipeline.Seriatim.Timeout)
if timeoutRaw != "" {
parsed, err := time.ParseDuration(timeoutRaw)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: parse seriatim timeout %q: %w", env.Config.Pipeline.Seriatim.Timeout, err)
}
timeout = parsed
}
for _, input := range rawInputs {
base := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input))
outPath := filepath.Join(normalizedDir, base+".normalized.json")
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stdout.log")
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stderr.log")
cfgPath := filepath.Join(paths.ConfigDir, "seriatim.normalize."+base+".generated.yml")
if runLayout.Enabled {
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize."+base+".stdout.log")
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize."+base+".stderr.log")
cfgPath = filepath.Join(runLayout.ConfigDir, "seriatim.normalize."+base+".generated.yml")
}
req := seriatim.NormalizeRequest{
Binary: env.Config.Pipeline.Seriatim.Binary,
InputTranscriptPath: input,
OutputNormalizedPath: outPath,
OutputSchema: env.Config.Pipeline.Seriatim.OutputSchema,
ReportPath: "",
StdoutLogPath: stdoutPath,
StderrLogPath: stderrPath,
GeneratedConfigPath: cfgPath,
Timeout: timeout,
}
res, err := env.Seriatim.Normalize(ctx, req)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: normalize input %q failed: %w", input, err)
}
finalOutputPath := outPath
if strings.TrimSpace(res.OutputNormalizedPath) != "" {
finalOutputPath = strings.TrimSpace(res.OutputNormalizedPath)
}
if err := validateTranscriptJSONFile(finalOutputPath); err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: normalized transcript %q invalid (input %q): %w", finalOutputPath, input, err)
}
normalizedInputs = append(normalizedInputs, finalOutputPath)
logs = append(logs, stdoutPath, stderrPath)
configs = append(configs, cfgPath)
meta = append(meta, normalizeMergeInputMeta{
InputPath: input,
OutputPath: finalOutputPath,
StdoutLogPath: stdoutPath,
StderrLogPath: stderrPath,
GeneratedConfig: cfgPath,
DurationMs: res.Duration.Milliseconds(),
ExitCode: res.ExitCode,
InvokedBinary: res.InvokedBinary,
OutputSchema: res.OutputSchema,
AdapterReportPath: res.ReportPath,
AdapterOutputPath: res.OutputNormalizedPath,
})
}
return normalizedInputs, logs, configs, meta, nil
}
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
fromManifest := make([]string, 0)
if m != nil && m.Stages != nil {

View File

@@ -2,6 +2,7 @@ package stage
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
@@ -14,9 +15,25 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type normalizeDirAssertingRunner struct {
*seriatim.FakeRunner
ExpectedDir string
}
func (r *normalizeDirAssertingRunner) Normalize(ctx context.Context, req seriatim.NormalizeRequest) (seriatim.NormalizeResult, error) {
info, err := os.Stat(r.ExpectedDir)
if err != nil {
return seriatim.NormalizeResult{}, fmt.Errorf("normalized directory check failed for %q: %w", r.ExpectedDir, err)
}
if !info.IsDir() {
return seriatim.NormalizeResult{}, fmt.Errorf("normalized path %q exists but is not a directory", r.ExpectedDir)
}
return r.FakeRunner.Normalize(ctx, req)
}
func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
inA := filepath.Join(paths.TranscriptsRawDir, "alice.json")
inB := filepath.Join(paths.TranscriptsRawDir, "bob.json")
@@ -54,6 +71,17 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if len(req.InputTranscriptPaths) != 2 {
t.Fatalf("input transcripts = %#v, want 2", req.InputTranscriptPaths)
}
if len(fake.NormalizeRequests) != 2 {
t.Fatalf("normalize requests = %#v, want 2", fake.NormalizeRequests)
}
if fake.NormalizeRequests[0].InputTranscriptPath != inA || fake.NormalizeRequests[1].InputTranscriptPath != inB {
t.Fatalf("normalize request inputs = %#v", fake.NormalizeRequests)
}
for _, mergeIn := range req.InputTranscriptPaths {
if !strings.Contains(mergeIn, filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("merge input path = %q, want normalized input path", mergeIn)
}
}
if len(result.Outputs) != 2 {
t.Fatalf("outputs len = %d, want 2", len(result.Outputs))
@@ -64,11 +92,11 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if result.Outputs[1].Kind != "seriatim_report" {
t.Fatalf("output[1] kind = %q, want seriatim_report", result.Outputs[1].Kind)
}
if len(result.Logs) != 2 {
t.Fatalf("logs = %#v, want 2 paths", result.Logs)
if len(result.Logs) != 6 {
t.Fatalf("logs = %#v, want 6 paths (4 normalize + 2 merge)", result.Logs)
}
if len(result.GeneratedConfigs) != 1 {
t.Fatalf("generated configs = %#v, want 1 path", result.GeneratedConfigs)
if len(result.GeneratedConfigs) != 3 {
t.Fatalf("generated configs = %#v, want 3 paths (2 normalize + 1 merge)", result.GeneratedConfigs)
}
meta := result.Metadata
@@ -84,11 +112,17 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if meta["input_transcripts_count"] != 2 {
t.Fatalf("metadata input_transcripts_count = %#v, want 2", meta["input_transcripts_count"])
}
if meta["normalized_inputs_count"] != 2 {
t.Fatalf("metadata normalized_inputs_count = %#v, want 2", meta["normalized_inputs_count"])
}
if _, ok := meta["normalize_inputs"]; !ok {
t.Fatalf("metadata normalize_inputs missing: %#v", meta)
}
}
func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
env.Seriatim = &seriatim.FakeRunner{}
@@ -104,7 +138,7 @@ func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
func TestMergeStageFailsOnInvalidInputJSON(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), "not-json")
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
@@ -121,7 +155,7 @@ func TestMergeStageFailsOnInvalidInputJSON(t *testing.T) {
func TestMergeStageFailsWhenAdapterFails(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
@@ -138,7 +172,7 @@ func TestMergeStageFailsWhenAdapterFails(t *testing.T) {
func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
@@ -152,11 +186,14 @@ func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testin
if len(fake.Requests) != 1 || len(fake.Requests[0].InputTranscriptPaths) != 1 {
t.Fatalf("fallback inputs = %#v", fake.Requests)
}
if len(fake.NormalizeRequests) != 1 {
t.Fatalf("normalize requests = %#v, want 1", fake.NormalizeRequests)
}
}
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
env, m := setupMergeEnvWithRelativeWorkspaceRoot(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, rawPath, `{"segments":[]}`)
@@ -180,14 +217,114 @@ func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t
if len(got) != 1 {
t.Fatalf("input transcript paths = %#v, want len 1", got)
}
if got[0] != filepath.Clean(rawPath) {
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
}
}
func TestMergeStageCreatesNormalizedRawDirectoryBeforeNormalize(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, rawPath, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
if err := os.RemoveAll(normalizedDir); err != nil {
t.Fatalf("remove normalized dir: %v", err)
}
env.Seriatim = &normalizeDirAssertingRunner{
FakeRunner: &seriatim.FakeRunner{},
ExpectedDir: normalizedDir,
}
if _, err := (mergeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("merge.Run() error = %v", err)
}
}
func TestMergeStageFailsWhenNormalizeAdapterFails(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, in, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
env.Seriatim = &seriatim.FakeRunner{NormalizeErr: context.DeadlineExceeded}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "normalize input") {
t.Fatalf("error = %q", err.Error())
}
}
func TestMergeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, in, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
badNormalized := filepath.Join(paths.ArtifactsDir, "bad.normalized.json")
writeFile(t, badNormalized, "not-json")
env.Seriatim = &seriatim.FakeRunner{
NormalizeResult: seriatim.NormalizeResult{
OutputNormalizedPath: badNormalized,
},
}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "normalized transcript") {
t.Fatalf("error = %q", err.Error())
}
}
func TestMergeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
env, m := setupMergeEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, in, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
fake := &seriatim.FakeRunner{}
env.Seriatim = fake
result, err := (mergeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("merge.Run() error = %v", err)
}
if len(fake.Requests) != 1 {
t.Fatalf("requests = %d, want 1", len(fake.Requests))
}
req := fake.Requests[0]
if !strings.Contains(req.OutputMergedTranscriptPath, filepath.Join("runs", m.RunID, "merge", "outputs")) {
t.Fatalf("run output path = %q, want run-local path", req.OutputMergedTranscriptPath)
}
if len(result.Outputs) == 0 {
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
}
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
}
}
func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, rawPath, `{"segments":[]}`)
@@ -211,8 +348,8 @@ func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
if len(got) != 1 {
t.Fatalf("input transcript paths = %#v, want len 1", got)
}
if got[0] != filepath.Clean(rawPath) {
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
}
}
@@ -269,7 +406,7 @@ func setupMergeEnvWithWorkspace(t *testing.T, workspace string) (*Env, *manifest
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}
return &Env{

View File

@@ -54,7 +54,11 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
return nil, fmt.Errorf("normalize: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "normalize")
if err != nil {
return nil, fmt.Errorf("normalize: resolve run-stage layout: %w", err)
}
processedPath, processedSource, err := discoverProcessedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("normalize: resolve processed transcript: %w", err)
@@ -67,18 +71,34 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
}
normalizeCfg := normalizeConfigOrDefault(env.Config.Pipeline.Normalize)
normalizedPath, err := resolveScriptoriumOutputPath(paths, normalizeCfg.OutputPath)
canonicalNormalizedPath, err := resolveScriptoriumOutputPath(paths, normalizeCfg.OutputPath)
if err != nil {
return nil, fmt.Errorf("normalize: resolve normalized output path: %w", err)
}
normalizedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalNormalizedPath)
if err != nil {
return nil, fmt.Errorf("normalize: resolve run-local normalized output path: %w", err)
}
reportEnabled := normalizeCfg.Report != nil && *normalizeCfg.Report
reportPath := ""
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "seriatim.normalize.report.json")
if reportEnabled {
reportPath = filepath.Join(paths.ArtifactsDir, "seriatim.normalize.report.json")
reportPath = canonicalReportPath
if runLayout.Enabled {
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
if err != nil {
return nil, fmt.Errorf("normalize: resolve run-local report path: %w", err)
}
}
}
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stdout.log")
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stderr.log")
generatedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.normalize.generated.yml")
if runLayout.Enabled {
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize.stdout.log")
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize.stderr.log")
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.normalize.generated.yml")
}
timeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
if err != nil {
return nil, fmt.Errorf("normalize: resolve seriatim timeout: %w", err)
@@ -113,44 +133,56 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
}
}
outputs := []artifacts.Ref{{
Kind: "transcript_normalized",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: finalNormalizedPath,
}}
promotedNormalized, err := promoteRunLocalOutput(env.ArtifactStore, finalNormalizedPath, canonicalNormalizedPath, artifacts.Ref{
Kind: "transcript_normalized",
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("normalize: promote normalized transcript: %w", err)
}
outputs := []artifacts.Ref{promotedNormalized}
if reportEnabled {
outputs = append(outputs, artifacts.Ref{
Kind: "seriatim_normalize_report",
Category: "artifacts",
SessionID: sessionID,
AbsolutePath: finalReportPath,
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
Kind: "seriatim_normalize_report",
Category: "artifacts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("normalize: promote report: %w", err)
}
outputs = append(outputs, promotedReport)
}
reportCanonicalPath := ""
if reportEnabled {
reportCanonicalPath = canonicalReportPath
}
meta := map[string]any{
"stage": "normalize",
"processed_transcript_path": processedPath,
"processed_transcript_source": processedSource,
"normalized_transcript_path": finalNormalizedPath,
"normalized_transcript_source": "stage.normalize.output",
"output_schema": normalizeCfg.OutputSchema,
"report_enabled": reportEnabled,
"report_path": finalReportPath,
"timeout": env.Config.Pipeline.Seriatim.Timeout,
"binary": env.Config.Pipeline.Seriatim.Binary,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"generated_config_path": generatedConfigPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_output_schema": res.OutputSchema,
"adapter_output_path": res.OutputNormalizedPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config": res.GeneratedConfigPath,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
"stage": "normalize",
"processed_transcript_path": processedPath,
"processed_transcript_source": processedSource,
"run_normalized_transcript_path": finalNormalizedPath,
"normalized_transcript_path": canonicalNormalizedPath,
"normalized_transcript_source": "stage.normalize.output",
"output_schema": normalizeCfg.OutputSchema,
"report_enabled": reportEnabled,
"run_report_path": finalReportPath,
"report_path": reportCanonicalPath,
"timeout": env.Config.Pipeline.Seriatim.Timeout,
"binary": env.Config.Pipeline.Seriatim.Binary,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"generated_config_path": generatedConfigPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_output_schema": res.OutputSchema,
"adapter_output_path": res.OutputNormalizedPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config": res.GeneratedConfigPath,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
}
if res.Metadata != nil {
meta["adapter_metadata"] = res.Metadata

View File

@@ -16,7 +16,7 @@ import (
func TestNormalizeStageConsumesProcessedTranscriptFromManifest(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
manifestProcessed := filepath.Join(paths.ArtifactsDir, "processed.from-manifest.json")
writeFile(t, manifestProcessed, `{"segments":[{"id":10}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":99}]}`)
@@ -43,7 +43,7 @@ func TestNormalizeStageConsumesProcessedTranscriptFromManifest(t *testing.T) {
func TestNormalizeStageFallsBackToProcessedTranscriptPath(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
fallback := filepath.Join(paths.TranscriptsDir, "processed.json")
writeFile(t, fallback, `{"segments":[{"id":1}]}`)
@@ -72,7 +72,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptMissing(t *testing.T) {
func TestNormalizeStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
env, m, _ := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), "not-json")
_, err := (normalizeStage{}).Run(context.Background(), env, m)
@@ -86,7 +86,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
func TestNormalizeStageFailsWhenProcessedTranscriptMissingSegments(t *testing.T) {
env, m, _ := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"schema":"audita.processed.v1"}`)
_, err := (normalizeStage{}).Run(context.Background(), env, m)
@@ -100,7 +100,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptMissingSegments(t *testing.T)
func TestNormalizeStagePassesConfiguredOutputSchemaToAdapter(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
env.Config.Pipeline.Normalize.OutputSchema = "seriatim-full"
@@ -121,7 +121,7 @@ func TestNormalizeStagePassesConfiguredOutputSchemaToAdapter(t *testing.T) {
func TestNormalizeStageRecordsNormalizedTranscriptOutputKind(t *testing.T) {
env, m, _ := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
result, err := (normalizeStage{}).Run(context.Background(), env, m)
@@ -138,7 +138,7 @@ func TestNormalizeStageRecordsNormalizedTranscriptOutputKind(t *testing.T) {
func TestNormalizeStageRecordsReportLogAndGeneratedConfigRefs(t *testing.T) {
env, m, _ := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
report := true
env.Config.Pipeline.Normalize.Report = &report
@@ -164,7 +164,7 @@ func TestNormalizeStageRecordsReportLogAndGeneratedConfigRefs(t *testing.T) {
func TestNormalizeStageFailsWhenAdapterReturnsError(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
ser.NormalizeErr = errors.New("normalize failed")
@@ -179,7 +179,7 @@ func TestNormalizeStageFailsWhenAdapterReturnsError(t *testing.T) {
func TestNormalizeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
badOutput := filepath.Join(paths.TranscriptsDir, "normalized.bad.json")
writeFile(t, badOutput, "not-json")
@@ -196,7 +196,7 @@ func TestNormalizeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
func TestNormalizeStageReportEnabledFailsWhenReportMissing(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
report := true
env.Config.Pipeline.Normalize.Report = &report
@@ -211,6 +211,33 @@ func TestNormalizeStageReportEnabledFailsWhenReportMissing(t *testing.T) {
}
}
func TestNormalizeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
env, m, ser := setupNormalizeEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
result, err := (normalizeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("normalize.Run() error = %v", err)
}
if len(ser.NormalizeRequests) != 1 {
t.Fatalf("normalize requests = %d, want 1", len(ser.NormalizeRequests))
}
req := ser.NormalizeRequests[0]
if !strings.Contains(req.OutputNormalizedPath, filepath.Join("runs", m.RunID, "normalize", "outputs")) {
t.Fatalf("run output path = %q, want run-local path", req.OutputNormalizedPath)
}
if len(result.Outputs) == 0 {
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
}
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
}
}
func setupNormalizeEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRunner) {
t.Helper()
workspace := t.TempDir()
@@ -243,7 +270,7 @@ func setupNormalizeEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRu
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}

View File

@@ -3,10 +3,8 @@ package stage
import (
"context"
"fmt"
"path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -47,25 +45,7 @@ func (s placeholderStage) Run(ctx context.Context, env *Env, m *manifest.Manifes
return result, nil
}
paths := env.ArtifactStore.SessionPaths(sessionID)
switch s.name {
case "archive":
if env.Storage != nil {
req := storage.ArchiveRequest{
SessionID: sessionID,
ManifestPath: paths.ManifestPath,
Items: []storage.ArchiveItem{{
Kind: "artifact",
LocalPath: filepath.Join(paths.ArtifactsDir, "session-log.md"),
RemoteKey: "sessions/" + sessionID + "/artifacts/session-log.md",
}},
}
_, err := env.Storage.Archive(ctx, req)
if err != nil {
return nil, fmt.Errorf("placeholder archive adapter call failed: %w", err)
}
}
case "notify":
if env.Notifier != nil {
req := notify.SendRequest{
@@ -95,7 +75,7 @@ func All() []Stage {
normalizeStage{},
trimStage{},
analyzeStage{},
placeholderStage{name: "archive"},
archiveStage{},
placeholderStage{name: "notify"},
}
}

View File

@@ -49,9 +49,22 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
Config: &config.Config{
SessionPath: sessionPath,
PipelinePath: pipelinePath,
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: root}},
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{
S3: &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
},
},
Archive: &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
},
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
@@ -66,10 +79,24 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
Audita: af,
Scriptorium: sc,
Storage: st,
ObjectStore: st,
Notifier: nf,
}
m := manifest.New("2026-05-03", time.Now().UTC())
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
m.RunID = "20260516T000000Z-abcdef12"
m.Campaign = "sample-campaign"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(root, "sample-campaign", "2026-05-03", m.RunID)
m.S3RunPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/" + m.RunID + "/"
m.S3Bucket = "my-dnd-archive"
if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "inputs"), 0o755); err != nil {
t.Fatalf("mkdir workdir inputs: %v", err)
}
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "inputs", "session.yml"), "session_id: 2026-05-03\n")
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "manifest.json"), "{}\n")
for _, s := range stages {
result, err := s.Run(context.Background(), env, m)
if err != nil {
@@ -150,6 +177,15 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
}
continue
}
if s.Name() == "archive" {
if result.Metadata["stage"] != "archive" {
t.Fatalf("archive metadata = %#v, want stage=archive", result.Metadata)
}
if result.Metadata["uploaded"] != true {
t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
}
continue
}
if result.Metadata["placeholder"] != true {
t.Fatalf("stage %q missing placeholder metadata", s.Name())
}
@@ -167,8 +203,11 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
if len(sc.RunRequests) != 0 {
t.Fatalf("scriptorium run calls = %d, want 0 when scriptorium config is absent", len(sc.RunRequests))
}
if len(st.Requests) != 1 {
t.Fatalf("storage calls = %d, want 1", len(st.Requests))
if len(st.Requests) != 0 {
t.Fatalf("storage archive calls = %d, want 0", len(st.Requests))
}
if _, ok := st.Objects["dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/"+m.RunID+"/manifest.json"]; !ok {
t.Fatalf("archive upload missing manifest key in fake object store")
}
if len(nf.Requests) != 1 {
t.Fatalf("notify calls = %d, want 1", len(nf.Requests))
@@ -181,13 +220,12 @@ func TestPlaceholderAdapterErrorPropagation(t *testing.T) {
env *Env
wantErr string
}{
{stageName: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("sterr")}}, wantErr: "archive"},
{stageName: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("nerr")}}, wantErr: "notify"},
}
root := t.TempDir()
store := artifacts.NewLocalStore(root)
_, err := store.EnsureLayout("2026-05-03")
_, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03")
if err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}

View File

@@ -55,7 +55,11 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
return nil, fmt.Errorf("polish: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "polish")
if err != nil {
return nil, fmt.Errorf("polish: resolve run-stage layout: %w", err)
}
mergedPath, source, err := discoverMergedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("polish: resolve merged transcript: %w", err)
@@ -72,12 +76,29 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
return nil, fmt.Errorf("polish: %w", err)
}
processedPath := filepath.Join(paths.TranscriptsDir, "processed.json")
reportPath := filepath.Join(paths.ArtifactsDir, "audita.report.json")
canonicalProcessedPath := filepath.Join(paths.TranscriptsDir, "processed.json")
processedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalProcessedPath)
if err != nil {
return nil, fmt.Errorf("polish: resolve run-local processed transcript path: %w", err)
}
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "audita.report.json")
reportPath := canonicalReportPath
if runLayout.Enabled {
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
if err != nil {
return nil, fmt.Errorf("polish: resolve run-local report path: %w", err)
}
}
workDir := filepath.Join(paths.ArtifactsDir, "audita-work")
stdoutPath := filepath.Join(paths.LogsDir, "audita.stdout.log")
stderrPath := filepath.Join(paths.LogsDir, "audita.stderr.log")
generatedConfigPath := filepath.Join(paths.ConfigDir, "audita.generated.yml")
if runLayout.Enabled {
workDir = filepath.Join(runLayout.ScratchDir, "audita-work")
stdoutPath = filepath.Join(runLayout.LogsDir, "audita.stdout.log")
stderrPath = filepath.Join(runLayout.LogsDir, "audita.stderr.log")
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "audita.generated.yml")
}
reportEnabled := env.Config.Pipeline.Audita.Report != nil && *env.Config.Pipeline.Audita.Report
req := audita.PolishRequest{
@@ -90,6 +111,12 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
Modules: append([]string(nil), env.Config.Pipeline.Audita.Modules...),
BaseURL: env.Config.Pipeline.Audita.BaseURL,
Model: env.Config.Pipeline.Audita.Model,
TranscriptDescription: env.Config.Pipeline.Audita.TranscriptDescription,
ConfigPath: env.Config.Pipeline.Audita.ConfigPath,
OutputSchema: env.Config.Pipeline.Audita.OutputSchema,
WorkDirRetention: env.Config.Pipeline.Audita.WorkDirRetention,
TotalLLMConcurrency: env.Config.Pipeline.Audita.TotalLLMConcurrency,
ProposalLLMConcurrency: env.Config.Pipeline.Audita.ProposalLLMConcurrency,
ValidationModel: env.Config.Pipeline.Audita.ValidationModel,
ValidationLLMConcurrency: env.Config.Pipeline.Audita.ValidationLLMConcurrency,
StdoutLogPath: stdoutPath,
@@ -122,63 +149,83 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
}
}
outputs := []artifacts.Ref{{
Kind: "transcript_processed",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: finalProcessedPath,
}}
promotedProcessed, err := promoteRunLocalOutput(env.ArtifactStore, finalProcessedPath, canonicalProcessedPath, artifacts.Ref{
Kind: "transcript_processed",
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("polish: promote processed transcript: %w", err)
}
outputs := []artifacts.Ref{promotedProcessed}
if reportEnabled {
outputs = append(outputs, artifacts.Ref{
Kind: "audita_report",
Category: "artifacts",
SessionID: sessionID,
AbsolutePath: finalReportPath,
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
Kind: "audita_report",
Category: "artifacts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("polish: promote report: %w", err)
}
outputs = append(outputs, promotedReport)
}
var validationConcurrency any
if env.Config.Pipeline.Audita.ValidationLLMConcurrency != nil {
validationConcurrency = *env.Config.Pipeline.Audita.ValidationLLMConcurrency
}
var llmConcurrency any
if env.Config.Pipeline.Audita.LLMConcurrency != nil {
llmConcurrency = *env.Config.Pipeline.Audita.LLMConcurrency
var totalLLMConcurrency any
if env.Config.Pipeline.Audita.TotalLLMConcurrency != nil {
totalLLMConcurrency = *env.Config.Pipeline.Audita.TotalLLMConcurrency
}
var proposalLLMConcurrency any
if env.Config.Pipeline.Audita.ProposalLLMConcurrency != nil {
proposalLLMConcurrency = *env.Config.Pipeline.Audita.ProposalLLMConcurrency
}
reportCanonicalPath := ""
if reportEnabled {
reportCanonicalPath = canonicalReportPath
}
meta := map[string]any{
"stage": "polish",
"merged_transcript_path": mergedPath,
"merged_transcript_source": source,
"glossary_path": glossaryPath,
"output_path": finalProcessedPath,
"report_path": finalReportPath,
"audita_work_dir": workDir,
"report_enabled": reportEnabled,
"modules": append([]string(nil), req.Modules...),
"base_url": req.BaseURL,
"model": req.Model,
"validation_model": req.ValidationModel,
"llm_concurrency": llmConcurrency,
"validation_llm_concurrency": validationConcurrency,
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"timeout": env.Config.Pipeline.Audita.Timeout,
"binary": env.Config.Pipeline.Audita.Binary,
"generated_config_path": generatedConfigPath,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_processed_output_path": res.ProcessedTranscriptPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config_path": res.GeneratedConfigPath,
"adapter_work_dir": res.WorkDir,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"credential_present": false,
"primary_llm_concurrency_via_env": false,
"stage": "polish",
"merged_transcript_path": mergedPath,
"merged_transcript_source": source,
"glossary_path": glossaryPath,
"run_output_path": finalProcessedPath,
"output_path": canonicalProcessedPath,
"run_report_path": finalReportPath,
"report_path": reportCanonicalPath,
"audita_work_dir": workDir,
"report_enabled": reportEnabled,
"modules": append([]string(nil), req.Modules...),
"base_url": req.BaseURL,
"model": req.Model,
"transcript_description": req.TranscriptDescription,
"config_path": req.ConfigPath,
"output_schema": req.OutputSchema,
"work_dir_retention": req.WorkDirRetention,
"validation_model": req.ValidationModel,
"total_llm_concurrency": totalLLMConcurrency,
"proposal_llm_concurrency": proposalLLMConcurrency,
"validation_llm_concurrency": validationConcurrency,
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"timeout": env.Config.Pipeline.Audita.Timeout,
"binary": env.Config.Pipeline.Audita.Binary,
"generated_config_path": generatedConfigPath,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_processed_output_path": res.ProcessedTranscriptPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config_path": res.GeneratedConfigPath,
"adapter_work_dir": res.WorkDir,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"credential_present": false,
}
if res.Metadata != nil {
meta["adapter_metadata"] = res.Metadata
@@ -188,9 +235,6 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
if value, ok := res.Metadata["credential_env_var"]; ok {
meta["credential_env_var"] = value
}
if value, ok := res.Metadata["primary_llm_concurrency_via_env"]; ok {
meta["primary_llm_concurrency_via_env"] = value
}
}
return &StageResult{

View File

@@ -16,7 +16,7 @@ import (
func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
writeFile(t, mergedPath, `{"segments":[]}`)
@@ -60,6 +60,24 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
if req.ValidationModel != "openrouter/google/gemma-4-31b-it" {
t.Fatalf("validation model = %q", req.ValidationModel)
}
if req.TranscriptDescription != "Campaign Session 42" {
t.Fatalf("transcript description = %q", req.TranscriptDescription)
}
if req.ConfigPath != "/etc/audita/config.yml" {
t.Fatalf("config path = %q", req.ConfigPath)
}
if req.OutputSchema != "audita-v1" {
t.Fatalf("output schema = %q", req.OutputSchema)
}
if req.WorkDirRetention != "auto" {
t.Fatalf("work dir retention = %q", req.WorkDirRetention)
}
if req.TotalLLMConcurrency == nil || *req.TotalLLMConcurrency != 3 {
t.Fatalf("total llm concurrency = %#v, want 3", req.TotalLLMConcurrency)
}
if req.ProposalLLMConcurrency == nil || *req.ProposalLLMConcurrency != 2 {
t.Fatalf("proposal llm concurrency = %#v, want 2", req.ProposalLLMConcurrency)
}
if req.ValidationLLMConcurrency == nil || *req.ValidationLLMConcurrency != 2 {
t.Fatalf("validation llm concurrency = %#v, want 2", req.ValidationLLMConcurrency)
}
@@ -89,11 +107,20 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
if result.Metadata["audita_work_dir"] != filepath.Join(paths.ArtifactsDir, "audita-work") {
t.Fatalf("metadata audita_work_dir = %#v", result.Metadata["audita_work_dir"])
}
if result.Metadata["total_llm_concurrency"] != 3 {
t.Fatalf("metadata total_llm_concurrency = %#v, want 3", result.Metadata["total_llm_concurrency"])
}
if result.Metadata["proposal_llm_concurrency"] != 2 {
t.Fatalf("metadata proposal_llm_concurrency = %#v, want 2", result.Metadata["proposal_llm_concurrency"])
}
if result.Metadata["output_schema"] != "audita-v1" {
t.Fatalf("metadata output_schema = %#v, want audita-v1", result.Metadata["output_schema"])
}
}
func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
writeFile(t, mergedPath, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
@@ -115,7 +142,7 @@ func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
func TestPolishStageFailsWhenMergedTranscriptMissing(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
env.Audita = &audita.FakeRunner{}
@@ -130,7 +157,7 @@ func TestPolishStageFailsWhenMergedTranscriptMissing(t *testing.T) {
func TestPolishStageFailsWhenMergedTranscriptInvalidJSON(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), "not-json")
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
env.Audita = &audita.FakeRunner{}
@@ -146,7 +173,7 @@ func TestPolishStageFailsWhenMergedTranscriptInvalidJSON(t *testing.T) {
func TestPolishStageFailsWhenGlossaryMissing(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
env.Audita = &audita.FakeRunner{}
@@ -161,7 +188,7 @@ func TestPolishStageFailsWhenGlossaryMissing(t *testing.T) {
func TestPolishStageFailsWhenAdapterFails(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
env.Audita = &audita.FakeRunner{Err: errors.New("audita failed")}
@@ -177,7 +204,7 @@ func TestPolishStageFailsWhenAdapterFails(t *testing.T) {
func TestPolishStageFailsWhenProcessedOutputInvalid(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
badPath := filepath.Join(paths.TranscriptsDir, "processed.invalid.json")
@@ -195,7 +222,7 @@ func TestPolishStageFailsWhenProcessedOutputInvalid(t *testing.T) {
func TestPolishStageFailsWhenReportInvalid(t *testing.T) {
env, m := setupPolishEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
badReport := filepath.Join(paths.ArtifactsDir, "bad.report.json")
@@ -211,6 +238,36 @@ func TestPolishStageFailsWhenReportInvalid(t *testing.T) {
}
}
func TestPolishStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
env, m := setupPolishEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
fake := &audita.FakeRunner{}
env.Audita = fake
result, err := (polishStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("polish.Run() error = %v", err)
}
if len(fake.Requests) != 1 {
t.Fatalf("requests = %d, want 1", len(fake.Requests))
}
req := fake.Requests[0]
if !strings.Contains(req.OutputProcessedPath, filepath.Join("runs", m.RunID, "polish", "outputs")) {
t.Fatalf("run output path = %q, want run-local path", req.OutputProcessedPath)
}
if len(result.Outputs) == 0 {
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
}
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
}
}
func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
t.Helper()
workspace := t.TempDir()
@@ -221,7 +278,8 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
report := true
llmConcurrency := 1
totalLLMConcurrency := 3
proposalLLMConcurrency := 2
validationLLMConcurrency := 2
cfg := &config.Config{
@@ -236,8 +294,13 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
Modules: []string{"glossary", "homophones", "grammar"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
TranscriptDescription: "Campaign Session 42",
ConfigPath: "/etc/audita/config.yml",
OutputSchema: "audita-v1",
WorkDirRetention: "auto",
ValidationModel: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
ValidationLLMConcurrency: &validationLLMConcurrency,
Report: &report,
},
@@ -251,7 +314,7 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}
return &Env{

View File

@@ -6,10 +6,12 @@ import (
"encoding/hex"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -33,7 +35,7 @@ func (prepareStage) Declares() IODecl {
}
}
func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("prepare: stage environment config is required")
}
@@ -52,7 +54,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
return nil, fmt.Errorf("prepare: session id is required")
}
paths, err := env.ArtifactStore.EnsureLayout(sessionID)
paths, err := ensureLayoutForEnv(env, sessionID)
if err != nil {
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
}
@@ -89,13 +91,12 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
}
}
resolvedAudio, err := resolveAudioFiles(sessionDir, env.Config.Session.Inputs)
resolvedLocalAudio, useS3Audio, err := resolveAudioInputs(sessionDir, env.Config.Session.Inputs)
if err != nil {
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
}
copiedByDest := map[string]string{}
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedAudio))
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
registerInput := func(kind, path, checksum string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
}
@@ -134,19 +135,14 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
registerInput(cfgFile.kind, cfgFile.dst, checksum)
}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return nil, fmt.Errorf("prepare: duplicate audio basename %q from %q and %q", base, prev, src)
if useS3Audio {
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize audio %q: %w", base, err)
} else {
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
registerInput("audio", dst, checksum)
}
sort.Slice(inputs, func(i, j int) bool {
@@ -162,7 +158,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
"prepared": true,
"stage": "prepare",
"inputs_count": len(inputs),
"audio_files_resolved": len(resolvedAudio),
"audio_files_resolved": countAudioInputs(inputs),
},
}, nil
}
@@ -171,7 +167,23 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
return yaml.Marshal(cfg)
}
func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
if inputs.AudioS3 != nil {
if hasLocal {
return nil, false, fmt.Errorf("audio_dir/audio_files and audio_s3 are mutually exclusive")
}
return nil, true, nil
}
local, err := resolveLocalAudioFiles(sessionDir, inputs)
if err != nil {
return nil, false, err
}
return local, false, nil
}
func resolveLocalAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
if len(inputs.AudioFiles) > 0 {
out := make([]string, 0, len(inputs.AudioFiles))
for _, p := range inputs.AudioFiles {
@@ -223,6 +235,146 @@ func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]
return out, nil
}
func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolvedAudio []string, registerInput func(kind, path, checksum string)) error {
copiedByDest := map[string]string{}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return fmt.Errorf("duplicate audio basename %q from %q and %q", base, prev, src)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return fmt.Errorf("materialize audio %q: %w", base, err)
}
registerInput("audio", dst, checksum)
}
return nil
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
if env.ObjectStore == nil {
return fmt.Errorf("s3 audio input requires object store backend")
}
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return fmt.Errorf("session campaign is required for s3 audio input")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return fmt.Errorf("run id is required for s3 audio input")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
objects, err := env.ObjectStore.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
for _, obj := range objects {
key := strings.TrimSpace(obj.Key)
if key == "" || strings.HasSuffix(key, "/") {
continue
}
if !isFlac(key) {
continue
}
audioObjects = append(audioObjects, obj)
}
sort.Slice(audioObjects, func(i, j int) bool {
return audioObjects[i].Key < audioObjects[j].Key
})
if len(audioObjects) == 0 {
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
}
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolAudioDir == "" {
spoolAudioDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, campaign, sessionID, runID)
}
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
}
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
}
seenBase := map[string]string{}
for _, obj := range audioObjects {
base := path.Base(obj.Key)
if prev, exists := seenBase[base]; exists && prev != obj.Key {
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
}
seenBase[base] = obj.Key
spoolPath := filepath.Join(spoolAudioDir, base)
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
}
workPath := filepath.Join(workAudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
if err != nil {
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
}
*inputs = append(*inputs, manifest.InputRecord{
Kind: "audio",
Path: workPath,
Checksum: checksum,
Source: "s3",
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
S3Key: obj.Key,
S3Size: obj.Size,
S3ETag: obj.ETag,
SpoolPath: spoolPath,
})
}
return nil
}
func countAudioInputs(inputs []manifest.InputRecord) int {
count := 0
for _, in := range inputs {
if in.Kind == "audio" {
count++
}
}
return count
}
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return ""
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return ""
}
if m != nil && strings.TrimSpace(m.LocalWorkDir) != "" {
return strings.TrimSpace(m.LocalWorkDir)
}
runID := ""
if m != nil {
runID = strings.TrimSpace(m.RunID)
}
if runID != "" {
return artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID)
}
return artifacts.SessionWorkDirForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID)
}
func resolvePath(baseDir, p string) (string, error) {
trimmed := strings.TrimSpace(p)
if trimmed == "" {

View File

@@ -8,6 +8,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -32,7 +33,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
t.Fatalf("result metadata = %#v, want prepared=true", result)
}
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
for _, p := range []string{
filepath.Join(paths.InputsDir, "session.yml"),
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
@@ -75,7 +76,7 @@ func TestPrepareStageAudioDirEnumeration(t *testing.T) {
t.Fatalf("audio_files_resolved = %#v, want 1", got)
}
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
if _, err := os.Stat(filepath.Join(paths.AudioDir, "a.flac")); err != nil {
t.Fatalf("expected copied flac: %v", err)
}
@@ -150,6 +151,139 @@ func TestPrepareStageIdempotent(t *testing.T) {
}
}
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/bob.FLAC", Data: []byte("bob")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/ignore.txt", Data: []byte("x")})
env.ObjectStore = fake
result, err := (prepareStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if result == nil || result.Metadata["audio_files_resolved"] != 2 {
t.Fatalf("result metadata = %#v, want audio_files_resolved=2", result)
}
aliceWork := filepath.Join(m.LocalWorkDir, "audio", "alice.flac")
bobWork := filepath.Join(m.LocalWorkDir, "audio", "bob.FLAC")
aliceSpool := filepath.Join(m.LocalSpoolDir, "alice.flac")
bobSpool := filepath.Join(m.LocalSpoolDir, "bob.FLAC")
for _, p := range []string{aliceWork, bobWork, aliceSpool, bobSpool} {
if _, err := os.Stat(p); err != nil {
t.Fatalf("expected file %q: %v", p, err)
}
}
audioInputs := 0
for _, in := range m.Inputs {
if in.Kind != "audio" {
continue
}
audioInputs++
if in.Source != "s3" {
t.Fatalf("audio input source = %q, want s3", in.Source)
}
if in.S3Bucket != "my-dnd-archive" {
t.Fatalf("audio input bucket = %q", in.S3Bucket)
}
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
t.Fatalf("audio input missing provenance: %#v", in)
}
}
if audioInputs != 2 {
t.Fatalf("audio input count = %d, want 2", audioInputs)
}
}
func TestPrepareStageS3AudioFailures(t *testing.T) {
tests := []struct {
name string
setup func(env *Env, m *manifest.Manifest, fake *storage.FakeBackend)
wantError string
}{
{
name: "no flac files",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/readme.txt", Data: []byte("x")})
},
wantError: "no .flac files found",
},
{
name: "list error",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.ListErr = os.ErrPermission
},
wantError: "list s3 audio objects",
},
{
name: "download error",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
fake.DownloadErr = os.ErrPermission
},
wantError: "download s3 audio object",
},
{
name: "basename collision",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/a/alice.flac", Data: []byte("a")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/b/alice.flac", Data: []byte("b")})
},
wantError: "duplicate s3 audio basename",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
tt.setup(env, m, fake)
env.ObjectStore = fake
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), tt.wantError) {
t.Fatalf("error = %v, want %q", err, tt.wantError)
}
})
}
}
func TestPrepareStageAudioSourceConflictFails(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioDir = "./audio"
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("error = %v, want mutually exclusive error", err)
}
}
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
t.Helper()
workspace := t.TempDir()
@@ -170,6 +304,7 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
PipelinePath: pipelinePath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",

136
internal/stage/run_local.go Normal file
View File

@@ -0,0 +1,136 @@
package stage
import (
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type runStageLayout struct {
Enabled bool
Root string
OutputsDir string
LogsDir string
ReportsDir string
ConfigDir string
ScratchDir string
}
func resolveRunStageLayout(
env *Env,
m *manifest.Manifest,
sessionPaths artifacts.SessionPaths,
sessionID, stageName string,
) (runStageLayout, error) {
if env == nil || env.Config == nil || env.Config.Pipeline == nil {
return runStageLayout{}, fmt.Errorf("stage environment pipeline config is required")
}
if strings.TrimSpace(sessionID) == "" {
return runStageLayout{}, fmt.Errorf("session id is required")
}
stageName = strings.TrimSpace(stageName)
if stageName == "" {
return runStageLayout{}, fmt.Errorf("stage name is required")
}
runID := ""
if m != nil {
runID = strings.TrimSpace(m.RunID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" && m != nil {
campaign = strings.TrimSpace(m.Campaign)
}
// Compatibility fallback for direct stage tests and older call paths
// that execute a stage without a run id.
if runID == "" || campaign == "" {
return runStageLayout{}, nil
}
root := artifacts.SessionRunStageDirForCampaign(
env.Config.Pipeline.Workspace.Root,
campaign,
sessionID,
runID,
stageName,
)
layout := runStageLayout{
Enabled: true,
Root: root,
OutputsDir: filepath.Join(root, "outputs"),
LogsDir: filepath.Join(root, "logs"),
ReportsDir: filepath.Join(root, "reports"),
ConfigDir: filepath.Join(root, "config"),
ScratchDir: filepath.Join(root, "scratch"),
}
for _, dir := range []string{
layout.Root,
layout.OutputsDir,
layout.LogsDir,
layout.ReportsDir,
layout.ConfigDir,
layout.ScratchDir,
} {
if err := os.MkdirAll(dir, 0o755); err != nil {
return runStageLayout{}, fmt.Errorf("create run-stage directory %q: %w", dir, err)
}
}
return layout, nil
}
func runLocalPathForCanonical(layout runStageLayout, sessionPaths artifacts.SessionPaths, canonicalPath string) (string, error) {
if !layout.Enabled {
return filepath.Clean(canonicalPath), nil
}
cleanCanonical := filepath.Clean(strings.TrimSpace(canonicalPath))
if cleanCanonical == "" {
return "", fmt.Errorf("canonical path is required")
}
rel, err := filepath.Rel(filepath.Clean(sessionPaths.Root), cleanCanonical)
if err != nil {
return "", fmt.Errorf("derive session-relative path for %q: %w", cleanCanonical, err)
}
rel = filepath.Clean(rel)
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("canonical path %q is outside session root %q", cleanCanonical, sessionPaths.Root)
}
localPath := filepath.Join(layout.OutputsDir, rel)
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return "", fmt.Errorf("create run-local output parent for %q: %w", localPath, err)
}
return localPath, nil
}
func promoteRunLocalOutput(
store artifacts.Store,
srcPath, canonicalPath string,
ref artifacts.Ref,
) (artifacts.Ref, error) {
srcPath = filepath.Clean(strings.TrimSpace(srcPath))
canonicalPath = filepath.Clean(strings.TrimSpace(canonicalPath))
if srcPath == "" {
return artifacts.Ref{}, fmt.Errorf("source path is required")
}
if canonicalPath == "" {
return artifacts.Ref{}, fmt.Errorf("canonical destination path is required")
}
data, err := os.ReadFile(srcPath)
if err != nil {
return artifacts.Ref{}, fmt.Errorf("read run-local output %q: %w", srcPath, err)
}
if err := store.WriteFileAtomic(canonicalPath, data, 0o644); err != nil {
return artifacts.Ref{}, fmt.Errorf("promote output to %q: %w", canonicalPath, err)
}
checksum, err := store.Checksum(canonicalPath)
if err != nil {
return artifacts.Ref{}, fmt.Errorf("checksum promoted output %q: %w", canonicalPath, err)
}
ref.AbsolutePath = canonicalPath
ref.Checksum = checksum
return ref, nil
}

View File

@@ -0,0 +1,35 @@
package stage
import (
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func TestRunLocalPathForCanonicalCreatesParentDirectories(t *testing.T) {
root := t.TempDir()
sessionRoot := filepath.Join(root, "work", "dilfs", "2026-05-17")
layout := runStageLayout{
Enabled: true,
OutputsDir: filepath.Join(sessionRoot, "runs", "run-1", "merge", "outputs"),
}
if err := os.MkdirAll(layout.OutputsDir, 0o755); err != nil {
t.Fatalf("mkdir outputs dir: %v", err)
}
canonical := filepath.Join(sessionRoot, "transcripts", "merged.json")
got, err := runLocalPathForCanonical(layout, artifacts.SessionPaths{Root: sessionRoot}, canonical)
if err != nil {
t.Fatalf("runLocalPathForCanonical() error = %v", err)
}
want := filepath.Join(layout.OutputsDir, "transcripts", "merged.json")
if got != want {
t.Fatalf("runLocalPathForCanonical() = %q, want %q", got, want)
}
if _, err := os.Stat(filepath.Dir(got)); err != nil {
t.Fatalf("expected run-local parent directory to exist: %v", err)
}
}

View File

@@ -0,0 +1,23 @@
package stage
import (
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
)
func sessionPathsForEnv(env *Env, sessionID string) artifacts.SessionPaths {
campaign := ""
if env != nil && env.Config != nil && env.Config.Session != nil {
campaign = strings.TrimSpace(env.Config.Session.Campaign)
}
return env.ArtifactStore.SessionPathsFor(campaign, sessionID)
}
func ensureLayoutForEnv(env *Env, sessionID string) (artifacts.SessionPaths, error) {
campaign := ""
if env != nil && env.Config != nil && env.Config.Session != nil {
campaign = strings.TrimSpace(env.Config.Session.Campaign)
}
return env.ArtifactStore.EnsureLayoutFor(campaign, sessionID)
}

View File

@@ -29,6 +29,7 @@ type Env struct {
Scriptorium scriptorium.Runner
Analyzer analyzer.Runner
Storage storage.Backend
ObjectStore storage.ObjectStore
Notifier notify.Sender
}

View File

@@ -55,7 +55,11 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
return nil, fmt.Errorf("transcribe: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "transcribe")
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-stage layout: %w", err)
}
audioFiles, err := discoverPreparedAudio(m, paths.AudioDir)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve audio inputs: %w", err)
@@ -88,10 +92,15 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
return nil, fmt.Errorf("transcribe: duplicate speaker/audio basename %q from %q and %q", base, prev, audioPath)
}
seenSpeaker[base] = audioPath
canonicalOut := filepath.Join(paths.TranscriptsRawDir, base+".json")
runOut, err := runLocalPathForCanonical(runLayout, paths, canonicalOut)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", base, err)
}
jobs = append(jobs, job{
speakerID: base,
audioPath: audioPath,
outPath: filepath.Join(paths.TranscriptsRawDir, base+".json"),
outPath: runOut,
})
}
@@ -197,12 +206,19 @@ dispatch:
sort.Strings(speakers)
outputs := make([]artifacts.Ref, 0, len(speakers))
runOutputPaths := make([]string, 0, len(speakers))
outputPaths := make([]string, 0, len(speakers))
orderedPerFile := make(map[string]any, len(speakers))
for _, speaker := range speakers {
ref := outputRef[speaker]
outputs = append(outputs, ref)
outputPaths = append(outputPaths, ref.AbsolutePath)
runOutputPaths = append(runOutputPaths, ref.AbsolutePath)
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
promoted, err := promoteRunLocalOutput(env.ArtifactStore, ref.AbsolutePath, canonicalOut, ref)
if err != nil {
return nil, fmt.Errorf("transcribe: promote %q output: %w", speaker, err)
}
outputs = append(outputs, promoted)
outputPaths = append(outputPaths, canonicalOut)
orderedPerFile[speaker] = perFile[speaker]
}
@@ -221,6 +237,7 @@ dispatch:
"retries": retries,
"retry_delay": env.Config.Pipeline.WhisperX.RetryDelay,
"timeout": env.Config.Pipeline.WhisperX.Timeout,
"run_output_paths": runOutputPaths,
"output_paths": outputPaths,
"per_file": orderedPerFile,
},

View File

@@ -53,8 +53,8 @@ func TestTranscribeStageTranscribesPreparedAudio(t *testing.T) {
}
sort.Strings(gotPaths)
wantPaths := []string{
filepath.Join(env.ArtifactStore.SessionPaths(m.SessionID).TranscriptsRawDir, "alice.json"),
filepath.Join(env.ArtifactStore.SessionPaths(m.SessionID).TranscriptsRawDir, "bob.json"),
filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json"),
filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "bob.json"),
}
sort.Strings(wantPaths)
if strings.Join(gotPaths, "|") != strings.Join(wantPaths, "|") {
@@ -191,6 +191,36 @@ func TestTranscribeStageInvalidJSONFails(t *testing.T) {
}
}
func TestTranscribeStageUsesRunLocalOutputAndPromotesCanonical(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
fake := &whisperx.FakeClient{}
env.WhisperX = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
result, err := (transcribeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("transcribe.Run() error = %v", err)
}
if len(fake.Requests) != 1 {
t.Fatalf("requests = %d, want 1", len(fake.Requests))
}
runOut := fake.Requests[0].OutputRawTranscriptPath
if !strings.Contains(runOut, filepath.Join("runs", m.RunID, "transcribe", "outputs")) {
t.Fatalf("run-local output path = %q, want runs/{run_id}/transcribe/outputs path", runOut)
}
if len(result.Outputs) != 1 {
t.Fatalf("outputs = %#v, want one output", result.Outputs)
}
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
}
}
func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Manifest) {
t.Helper()
@@ -203,7 +233,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
@@ -227,6 +257,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",

View File

@@ -54,7 +54,11 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
return nil, fmt.Errorf("trim: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "trim")
if err != nil {
return nil, fmt.Errorf("trim: resolve run-stage layout: %w", err)
}
normalizedPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("trim: resolve normalized transcript: %w", err)
@@ -69,10 +73,14 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
trimCfg := env.Config.Pipeline.Trim
enabled := trimCfg != nil && trimCfg.Enabled
trimmedPath, err := resolveTrimmedOutputPath(paths, trimCfg)
canonicalTrimmedPath, err := resolveTrimmedOutputPath(paths, trimCfg)
if err != nil {
return nil, fmt.Errorf("trim: resolve trimmed output path: %w", err)
}
trimmedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalTrimmedPath)
if err != nil {
return nil, fmt.Errorf("trim: resolve run-local trimmed output path: %w", err)
}
logPaths := []string{}
generatedConfigs := []string{}
@@ -81,7 +89,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
"trim_enabled": enabled,
"normalized_transcript_path": normalizedPath,
"normalized_transcript_source": normalizedSource,
"trimmed_output_path": trimmedPath,
"run_trimmed_output_path": trimmedPath,
"trimmed_output_path": canonicalTrimmedPath,
}
if !enabled {
@@ -91,14 +100,17 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
}
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
Kind: "transcript_trimmed",
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
}
metadata["trim_action"] = "copy_disabled"
return &StageResult{
Outputs: []artifacts.Ref{{
Kind: "transcript_trimmed",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: trimmedPath,
}},
Outputs: []artifacts.Ref{promotedTrimmed},
Metadata: metadata,
}, nil
}
@@ -114,13 +126,22 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
}
boundsCfg := trimCfg.Bounds
boundsOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.OutputPath)
canonicalBoundsOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.OutputPath)
if err != nil {
return nil, fmt.Errorf("trim: resolve bounds output path: %w", err)
}
boundsOutputPath, err := runLocalPathForCanonical(runLayout, paths, canonicalBoundsOutputPath)
if err != nil {
return nil, fmt.Errorf("trim: resolve run-local bounds output path: %w", err)
}
boundsStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stdout.log")
boundsStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stderr.log")
boundsGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.generated.yml")
if runLayout.Enabled {
boundsStdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.stdout.log")
boundsStderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.stderr.log")
boundsGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium.bounds.generated.yml")
}
boundsTimeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, boundsCfg.Timeout)
if err != nil {
return nil, fmt.Errorf("trim: resolve bounds timeout: %w", err)
@@ -133,7 +154,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
metadata["bounds_prompt_id"] = boundsCfg.PromptID
metadata["bounds_profile_id"] = boundsCfg.ProfileID
metadata["bounds_output_path"] = boundsOutputPath
metadata["run_bounds_output_path"] = boundsOutputPath
metadata["bounds_output_path"] = canonicalBoundsOutputPath
metadata["bounds_timeout"] = boundsTimeout.String()
metadata["bounds_input_name"] = boundsCfg.TranscriptInputName
metadata["bounds_input_path"] = normalizedPath
@@ -141,13 +163,22 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
renderOutputPath := ""
if boundsCfg.RenderDebug {
renderOutputPath, err = resolveScriptoriumOutputPath(paths, boundsCfg.RenderOutputPath)
canonicalRenderOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.RenderOutputPath)
if err != nil {
return nil, fmt.Errorf("trim: resolve bounds render output path: %w", err)
}
renderOutputPath, err = runLocalPathForCanonical(runLayout, paths, canonicalRenderOutputPath)
if err != nil {
return nil, fmt.Errorf("trim: resolve run-local bounds render output path: %w", err)
}
renderStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stdout.log")
renderStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stderr.log")
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.render.generated.yml")
if runLayout.Enabled {
renderStdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.render.stdout.log")
renderStderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.render.stderr.log")
renderGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium.bounds.render.generated.yml")
}
renderReq := scriptorium.RenderArtifactRequest{
Binary: env.Config.Pipeline.Scriptorium.Binary,
@@ -254,7 +285,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
metadata["end_segment_id"] = boundsPayload.EndSegmentID
metadata["warnings"] = boundsPayload.Warnings
metadata["keep_selector"] = keepSelector
metadata["bounds_output_path"] = finalBoundsOutputPath
metadata["run_bounds_output_path"] = finalBoundsOutputPath
metadata["bounds_stdout_log_path"] = boundsStdoutLogPath
metadata["bounds_stderr_log_path"] = boundsStderrLogPath
metadata["bounds_generated_config_path"] = boundsGeneratedConfigPath
@@ -279,6 +310,11 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
trimStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stdout.log")
trimStderrLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stderr.log")
trimGeneratedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.trim.generated.yml")
if runLayout.Enabled {
trimStdoutLogPath = filepath.Join(runLayout.LogsDir, "seriatim.trim.stdout.log")
trimStderrLogPath = filepath.Join(runLayout.LogsDir, "seriatim.trim.stderr.log")
trimGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.trim.generated.yml")
}
trimTimeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
if err != nil {
return nil, fmt.Errorf("trim: resolve seriatim timeout: %w", err)
@@ -315,23 +351,25 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
return nil, fmt.Errorf("trim: trimmed transcript %q invalid: %w", trimmedPath, err)
}
outputs := []artifacts.Ref{
{
Kind: "transcript_trimmed",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: trimmedPath,
},
{
Kind: "session_bounds",
Category: "artifacts",
SessionID: sessionID,
AbsolutePath: finalBoundsOutputPath,
},
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
Kind: "transcript_trimmed",
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
}
promotedBounds, err := promoteRunLocalOutput(env.ArtifactStore, finalBoundsOutputPath, canonicalBoundsOutputPath, artifacts.Ref{
Kind: "session_bounds",
Category: "artifacts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("trim: promote session bounds: %w", err)
}
return &StageResult{
Outputs: outputs,
Outputs: []artifacts.Ref{promotedTrimmed, promotedBounds},
Logs: logPaths,
GeneratedConfigs: generatedConfigs,
Metadata: metadata,

View File

@@ -18,7 +18,7 @@ import (
func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeFile(t, normalized, `{"segments":[{"id":10},{"id":868}]}`)
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
@@ -70,7 +70,7 @@ func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11}`
@@ -94,7 +94,7 @@ func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
@@ -114,7 +114,7 @@ func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
func TestTrimStageRenderDebugDiagnosticsAreNotStageOutputs(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
@@ -152,7 +152,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptMissing(t *testing.T) {
func TestTrimStageDoesNotFallBackToProcessedTranscript(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -172,7 +172,7 @@ func TestTrimStageDoesNotFallBackToProcessedTranscript(t *testing.T) {
func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
env, m, _, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), "not-json")
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -185,7 +185,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T) {
env, m, _, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"schema":"audita.processed.v1"}`)
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -198,7 +198,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T
func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = "not-json"
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -212,7 +212,7 @@ func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":2,"end_segment_id":1}`
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -226,7 +226,7 @@ func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":20},{"id":21}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":21}`
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -240,7 +240,7 @@ func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.RunErr = errors.New("bounds failed")
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -254,7 +254,7 @@ func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
ser.TrimErr = errors.New("trim failed")
@@ -269,7 +269,7 @@ func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
paths := sessionPathsForEnv(env, m.SessionID)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
normalizedBody := `{"segments":[{"id":1,"text":"alpha"},{"id":2,"text":"beta"}]}`
writeFile(t, normalized, normalizedBody)
@@ -298,6 +298,39 @@ func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
}
}
func TestTrimStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11,"warnings":[]}`
result, err := (trimStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("trim.Run() error = %v", err)
}
if len(scr.RunRequests) != 1 || len(ser.TrimRequests) != 1 {
t.Fatalf("scriptorium run=%d seriatim trim=%d, want 1/1", len(scr.RunRequests), len(ser.TrimRequests))
}
if !strings.Contains(scr.RunRequests[0].OutputPath, filepath.Join("runs", m.RunID, "trim", "outputs")) {
t.Fatalf("bounds run output path = %q, want run-local path", scr.RunRequests[0].OutputPath)
}
if !strings.Contains(ser.TrimRequests[0].OutputTrimmedPath, filepath.Join("runs", m.RunID, "trim", "outputs")) {
t.Fatalf("trim output path = %q, want run-local path", ser.TrimRequests[0].OutputTrimmedPath)
}
if len(result.Outputs) < 2 {
t.Fatalf("outputs = %#v, want promoted trimmed+bounds outputs", result.Outputs)
}
for _, out := range result.Outputs {
if strings.Contains(out.AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("promoted output path = %q, want canonical session path", out.AbsolutePath)
}
}
}
type boundsScriptoriumRunner struct {
RunRequests []scriptorium.RunArtifactRequest
RenderRequests []scriptorium.RenderArtifactRequest
@@ -430,7 +463,7 @@ func setupTrimEnv(t *testing.T) (*Env, *manifest.Manifest, *boundsScriptoriumRun
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}