26 Commits

Author SHA1 Message Date
e58e545686 Audit workspace architecture implementation plan
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-17 13:22:09 -05:00
6ff54c5a0f Documentation update and reorganization 2026-05-17 13:14:28 -05:00
924b5d15c6 Applied a more general bugfix to path-resolution issues in the archive stage 2026-05-17 11:08:11 -05:00
b065663180 Bugfix involving path resolution in the archive stage 2026-05-17 11:03:02 -05:00
3ba564b00f Bugfix involving directory creation during the merge stage 2026-05-17 08:18:57 -05:00
a3986cf0d6 Centralized defaults into internal/config/defaults.go 2026-05-17 07:53:51 -05:00
539601bd16 Updated the merge stage to normalize the per-speaker transcripts before merging them
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 23:30:13 -05:00
6ca1c8d6b0 The backend S3 client now resolves credentials from user-configurable environment variables 2026-05-16 23:22:21 -05:00
4b7b50981b Add .gocache to .gitignore and minor documentation cleanup 2026-05-16 23:21:45 -05:00
33f7ae8f2e Simplify downstream tool configuration
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 23:30:40 +00:00
d5a9ad38f8 Add post-archive cleanup policies 2026-05-16 23:09:39 +00:00
6fbefb9867 Add session discovery and template support 2026-05-16 22:57:42 +00:00
1665359486 Added a locally generated UX progress report 2026-05-16 20:11:20 +00:00
03f2543927 Created a UX status report 2026-05-16 20:09:53 +00:00
fe9c348092 Document and review S3 archive workflow 2026-05-16 15:24:44 +00:00
f7f8f1a949 Promote current session artifacts to storage 2026-05-16 15:01:01 +00:00
d40c91acde Upload successful run records to storage 2026-05-16 14:43:29 +00:00
ed4dcf1ef7 Updated go.mod 2026-05-16 09:34:43 -05:00
24cce49a70 Download S3 audio during prepare 2026-05-16 14:33:42 +00:00
1e6db89dd4 Add remote storage backend 2026-05-16 14:22:04 +00:00
0454296c81 Add archive storage path configuration 2026-05-16 14:11:59 +00:00
58c6ab2d54 Updated audita configuration to reflect the new audita public CLI
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 08:46:48 -05:00
7995c41675 Update the audita integration documentation reference
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 08:02:18 -05:00
62551d43a0 Added filesystem-based secrets loading configuration 2026-05-16 07:56:22 -05:00
8395c12dd3 Update documentation to include an implementation roadmap for the archive stage 2026-05-16 07:36:25 -05:00
dc8e1040f2 Rationalize config file locations and update documentation
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-14 21:02:06 -05:00
71 changed files with 7902 additions and 595 deletions

5
.gitignore vendored
View File

@@ -2,6 +2,8 @@
.codex
AGENTS.md
.DS_Store
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -22,6 +24,9 @@ AGENTS.md
# Dependency directories (remove the comment below to include it)
# vendor/
# Go cache
.gocache
# Go workspace file
go.work
go.work.sum

188
README.md
View File

@@ -16,7 +16,6 @@ Implemented now:
Not implemented yet:
- `archive` stage behavior
- `notify` stage behavior
- additional analyze artifacts beyond `session_recap`
- generic DAG orchestration
@@ -28,8 +27,127 @@ Narratio expects two YAML files:
- `pipeline.yml`: pipeline/workspace settings
- `session.yml`: per-session settings
Pipeline config lookup for CLI commands:
- if `--config <path>` is provided, Narratio uses that path
- if `--config` is omitted, Narratio searches in this order:
- `/usr/local/etc/narratio/pipeline.yml`
- `/etc/narratio/pipeline.yml`
Session config lookup for CLI commands:
- if `--session <path>` is provided, Narratio uses that path
- if `--session` is omitted, Narratio searches in this order:
- `./session.yml`
- `/usr/local/etc/narratio/session.yml`
- `/etc/narratio/session.yml`
Session template support:
- Narratio renders `session.yml` templates before strict YAML decode.
- `--session-id <value>` provides the `session_id` template variable.
- Supported placeholder forms:
- `{{session_id}}`
- `{{ session_id }}`
- unresolved template placeholders fail with a clear error.
- strict YAML validation still runs after rendering.
- concrete `session.yml` files without templates remain fully supported.
Optional secrets-from-files config:
- `pipeline.secrets.env_dir` may point to a directory of secret files
- each top-level file with an env-var-style name is loaded as an environment variable:
- file name = env var name
- file contents = env var value (trailing newline/CRLF trimmed)
- process environment wins: existing env vars are not overwritten
- if configured, Narratio fails fast when `env_dir` is missing/unreadable
- relative `env_dir` values resolve from Narratios current working directory
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
Maintainer note: application defaults are centralized in [`internal/config/defaults.go`](internal/config/defaults.go).
## Storage And Archive Foundations
Narratio now includes configuration and path-model foundations for archive support, plus implemented prepare-stage S3 audio input.
Implemented foundations:
- `pipeline.storage.s3` config shape (`bucket`, `root_prefix`, `region`, `endpoint`, `force_path_style`, `access_key_id_env`, `secret_access_key_env`)
- `pipeline.spool` config shape (`root`, `delete_audio_after_archive`)
- `pipeline.archive` config shape (`enabled`, `upload_run`, `promote_artifacts`)
- promotion-rule validation (`from`/`to` required, relative-only paths, traversal rejected)
- `session.campaign` requirement for campaign-aware path construction
- optional `session.inputs.audio_s3.prefix` modeling and prepare-stage S3 audio download
- run ID generation and S3/local path helper foundations
- manifest run/path identity fields
Current defaults:
- `pipeline.storage.s3.root_prefix`: `dnd`
- `pipeline.storage.s3.access_key_id_env`: `OBJECT_STORAGE_KEY_ID`
- `pipeline.storage.s3.secret_access_key_env`: `OBJECT_STORAGE_KEY`
- `pipeline.workspace.cleanup_after_archive`: `false`
- `pipeline.spool.root`: `/var/spool/narratio`
- `pipeline.spool.delete_audio_after_archive`: `false`
- `pipeline.archive.enabled`: `true`
- `pipeline.archive.upload_run`: `true`
- default `pipeline.archive.promote_artifacts`:
- `transcripts/trimmed.json` -> `transcripts/trimmed.json` (`required: true`)
- `artifacts/session_recap.md` -> `artifacts/session_recap.md` (`required: true`)
Current boundaries:
- local development audio (`audio_dir` / `audio_files`) still works
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
- real S3-compatible backend now exists in the storage adapter package
- storage backend tests use fake storage and do not require live S3
- archive uploads successful run records under `runs/{run_id}/`
- archive does not upload local audio by default
- archive uploads promoted outputs to session-level keys using `archive.promote_artifacts`
- archive uploads `current/manifest.json`
- archive uploads `current/run_id.txt` last as the effective commit marker
- required missing promotions fail archive
- optional missing promotions are skipped and recorded
- cleanup remains conservative and opt-in:
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory after successful archive commit
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir after successful archive commit
- cleanup executes only after all selected stages for the command invocation succeed
- cleanup does not run for failed, incomplete, skipped, or unarchived runs
- local development `audio_dir`/`audio_files` source inputs are never deleted by spool cleanup
- S3 credentials are resolved from configured env-var names when both are present; if either is missing, Narratio falls back to the AWS SDK default credential chain
S3 input details and current boundaries are documented in [docs/s3-audio-input.md](docs/s3-audio-input.md).
## Remote Storage Backend
Narratio includes an object-store backend layer for future prepare/archive work:
- `List(ctx, prefix)`
- `Download(ctx, key, localPath)`
- `Upload(ctx, localPath, key, opts)`
- `Exists(ctx, key)`
Implemented backends:
- fake storage backend for deterministic tests
- S3-compatible backend built from `pipeline.storage.s3`
Key invariant:
- callers pass full bucket-relative object keys
- storage backends do not prepend `root_prefix` and do not infer session/campaign paths
Current boundary:
- `prepare` uses `List` + `Download` through the backend when `session.inputs.audio_s3` is configured
- `archive` uses `Upload` through the backend for successful run-record uploads
- `archive` also uses `Upload` for promotion writes and current pointers
- no failed or incomplete runs are uploaded
- local audio is not re-uploaded by default
Archive run-upload details and boundaries are documented in [docs/archive-storage.md](docs/archive-storage.md).
## Canonical Stage Order
1. `prepare`
@@ -49,6 +167,52 @@ YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
## Seriatim Configuration
`pipeline.seriatim` configures the Seriatim subprocess adapter used by `merge`, `normalize`, and `trim`.
Minimal behavior:
- `pipeline.seriatim` may be omitted entirely.
- when omitted, Narratio defaults to:
- `binary: seriatim`
- `timeout: 10m`
- `output_schema: seriatim-intermediate`
- `coalesce_gap: 3.0`
- `report: true`
Optional overrides in `pipeline.seriatim` continue to work, including explicit binary paths and advanced `env` tuning values.
## Audita Configuration
`pipeline.audita` configures the real Audita subprocess adapter used by `polish`.
Minimal behavior:
- `pipeline.audita` may be omitted entirely.
- when omitted, Narratio defaults to:
- `binary: audita`
- `timeout: 3h`
- `report: true`
Optional:
- `llm_api_key_env` (when set, Narratio requires that env var and passes it to Audita as `AUDITA_LLM_API_KEY`)
- `modules` override list (when empty/omitted, Narratio does not pass `--modules`)
- `base_url` (when omitted, Narratio does not pass `--base-url`; Audita runtime defaults/config may apply)
- `model` (when omitted, Narratio does not pass `--model`; Audita runtime defaults/config may apply)
- `transcript_description`
- `config_path`
- `output_schema` (`bare-segments` or `audita-v1`)
- `work_dir_retention` (`always`, `auto`, or `never`)
- `total_llm_concurrency` (> 0 when provided)
- `proposal_llm_concurrency` (> 0 when provided)
- `validation_model`
- `validation_llm_concurrency` (> 0 when provided)
- `report` (defaults to `true`)
Narratio passes only configured optional Audita flags. Omitted optional values are left to Audita runtime defaults/config.
## Normalize Configuration
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
@@ -142,7 +306,7 @@ Render-debug files are diagnostics and are not treated as canonical stage output
Key points:
- `scriptorium.binary` is required when section is present
- `scriptorium.binary` defaults to `scriptorium` when section is present
- `scriptorium.config_path` is optional
- `scriptorium.timeout` defaults to `10m` when omitted
- `scriptorium.render_debug` enables render diagnostics globally
@@ -192,6 +356,8 @@ Prompt IDs and profile IDs are configuration values. They are not hardcoded in a
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs.
## Scriptorium Runtime Behavior
Narratio integrates with Scriptorium through the public CLI subprocess contract:
@@ -228,7 +394,9 @@ Expected session output paths:
Starter files:
- `examples/pipeline.minimal.yml`
- `examples/pipeline.audita-overrides.yml`
- `examples/session.minimal.yml`
- `examples/session.template.yml`
- `examples/speakers.yml`
## Commands
@@ -242,7 +410,15 @@ go test ./...
Plan a run:
```bash
go run ./cmd/narratio plan --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
go run ./cmd/narratio plan --session examples/session.minimal.yml
```
Use `--config <path>` to override default pipeline lookup when needed.
Run with a discoverable session template:
```bash
go run ./cmd/narratio run --session-id 2026-04-04
```
Run full pipeline:
@@ -257,6 +433,12 @@ Run analyze only:
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
```
Resume with a template session ID:
```bash
go run ./cmd/narratio resume --config examples/pipeline.minimal.yml --session examples/session.template.yml --session-id 2026-04-04
```
## Operational Note
Checksum-based stale detection is not implemented yet.

View File

@@ -22,10 +22,24 @@ Implemented:
- real `trim` stage producing `transcripts/trimmed.json`
- real `analyze` stage for initial `session_recap` generation
- optional Scriptorium render diagnostics (`render_debug`) before production run
- storage/archive configuration and validation foundations for:
- `pipeline.storage.s3`
- `pipeline.spool`
- `pipeline.archive` promotion rules
- `session.inputs.audio_s3`
- run identity and path-model foundations:
- run ID generation (`YYYYMMDDTHHMMSSZ-xxxxxxxx`)
- S3 session/run/current key builders
- campaign/session/run local work/spool path helpers
- manifest run/path identity fields (`campaign`, `run_id`, local and S3 prefixes)
- remote storage backend layer:
- narrow object-store interface (`List`, `Download`, `Upload`, `Exists`)
- fake storage backend for deterministic tests (no network dependency)
- S3-compatible backend using AWS SDK v2
- config-based object-store construction helper
Still placeholder/future:
- `archive` stage behavior
- `notify` stage behavior
- additional Scriptorium artifact types beyond `session_recap`
- artifact-to-artifact workflows beyond the initial single-artifact implementation
@@ -94,12 +108,183 @@ Adapter behavior:
## 5. Configuration Contract
CLI pipeline config path resolution:
- when `--config <path>` is provided, that path is used
- when `--config` is omitted, Narratio searches defaults in order:
- `/usr/local/etc/narratio/pipeline.yml`
- `/etc/narratio/pipeline.yml`
- default values are centralized in `internal/config/defaults.go`
CLI session config path resolution:
- when `--session <path>` is provided, that path is used
- when `--session` is omitted, Narratio searches defaults in order:
- `./session.yml`
- `/usr/local/etc/narratio/session.yml`
- `/etc/narratio/session.yml`
Session template rendering:
- session templates are rendered before strict YAML decode
- `--session-id <value>` provides the `session_id` template variable
- supported placeholders:
- `{{session_id}}`
- `{{ session_id }}`
- unresolved placeholders fail clearly
- strict `KnownFields(true)` YAML validation still applies after rendering
- if rendered `session.session_id` conflicts with `--session-id`, load fails clearly
Optional pipeline secrets directory:
- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution
- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed)
- only env-var-style file names are considered; other entries are ignored
- existing process environment values are preserved (not overwritten)
- if configured, unreadable/missing `env_dir` fails command execution early
- relative `env_dir` values are resolved from current working directory
Storage and archive foundations:
- `pipeline.storage.s3` is available for modeling S3 coordinates:
- `bucket`
- `root_prefix` (default `dnd`)
- `region`
- `endpoint`
- `force_path_style` (default `false`)
- `access_key_id_env` (default `OBJECT_STORAGE_KEY_ID`)
- `secret_access_key_env` (default `OBJECT_STORAGE_KEY`)
- `pipeline.spool.root` defaults to `/var/spool/narratio`
- `pipeline.workspace.cleanup_after_archive` defaults to `false`
- `pipeline.spool.delete_audio_after_archive` defaults to `false`
- `pipeline.archive` is optional and defaults to:
- `enabled: true`
- `upload_run: true`
- default `promote_artifacts`:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
- archive promotion rules enforce safe relative paths:
- `from` and `to` are required
- absolute paths are rejected
- traversal segments such as `..` are rejected
Session input foundations:
- `session.campaign` is required
- local audio remains supported through `session.inputs.audio_dir` or `session.inputs.audio_files`
- optional S3 audio input shape is `session.inputs.audio_s3.prefix`
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
- when `audio_s3` is configured, `prepare` lists and downloads `.flac` objects through the object-store backend
Cross-config validation scope:
- `pipeline.storage.s3.bucket` is required only when an S3-dependent feature is explicitly configured (for current foundations, that includes `session.inputs.audio_s3`, and archive upload intent when using `storage.backend: s3`)
- no AWS credential values are stored in Narratio config; only env-var names are configured
- when both configured credential env vars resolve to non-empty values, the S3 backend uses them as static credentials
- when either configured credential value is missing, the S3 backend falls back to the AWS SDK default credential chain
Remote object-store backend scope:
- remote storage APIs are isolated to `internal/adapters/storage`
- AWS SDK types remain contained within the S3 backend implementation package
- S3 key/session path semantics remain outside the backend, with this invariant:
- callers pass full bucket-relative object keys
- backend methods do not prepend `root_prefix` or infer campaign/session/run paths
- `prepare` now uses object-store `List` and `Download` for S3 audio input
- `archive` now uses object-store `Upload` for successful run-record upload under the run prefix
- `archive` now uses object-store `Upload` for promoted outputs and current pointers
Prepare S3 audio behavior (implemented):
- compute session prefix as `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
- resolve `session.inputs.audio_s3.prefix` under that session prefix
- list objects under the computed audio prefix and filter `.flac` keys
- fail clearly when no `.flac` objects are found
- download selected objects to spool audio path:
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
- materialize audio files into workdir audio path:
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
- record S3 provenance in manifest input records (bucket/key/metadata/local paths/checksum)
- no AWS SDK types are used in stage code; storage implementation details stay in storage adapter packages
Archive publishing behavior (implemented):
- `archive` verifies prerequisite stage success before upload:
- `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`
- only successful/completed runs are uploaded
- uploaded run record destination is:
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
- uploaded existing local paths include:
- `inputs/`, `transcripts/`, `artifacts/`, optional `reports/`, `config/`, `logs/`, and `manifest.json`
- local `audio/` is intentionally excluded from upload by default
- file upload order is deterministic (sorted relative paths)
- `archive.enabled: false` and `archive.upload_run: false` skip upload cleanly
- stage metadata records non-secret upload context:
- run upload details, promoted output details, current manifest key, current pointer key
- no secrets, transcript contents, prompt contents, or environment dumps
- promotion rules:
- `from` resolves from local workdir
- `to` resolves under session-level S3 root
- missing required source fails archive
- missing optional source is skipped and recorded
- default promoted outputs:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
- current pointers:
- `current/manifest.json` uploaded after run upload and promotions
- `current/run_id.txt` uploaded last with `{run_id}\n`
- `current/run_id.txt` is the effective commit marker
- if promotion or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`
- failed/incomplete runs remain local and are not uploaded
- post-archive local cleanup (implemented, opt-in):
- cleanup runs only after archive succeeded and wrote `current/run_id.txt`
- cleanup is executed after all selected stages in the command invocation succeed (for example, a later `notify` failure leaves local files intact)
- `pipeline.spool.delete_audio_after_archive: true` removes only `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
- `pipeline.workspace.cleanup_after_archive: true` removes only `{workspace.root}/work/{campaign}/{session_id}/{run_id}/`
- cleanup does not run when archive is skipped/disabled/fails or when run upload is disabled
- local development `audio_dir`/`audio_files` inputs are never removed by spool cleanup
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
`pipeline.audita` drives the real Audita subprocess adapter for the `polish` stage.
Audita defaulted fields:
- `binary` defaults to `audita`
- `timeout` defaults to `3h`
- `report` defaults to `true`
Audita optional fields:
- `llm_api_key_env` (enforced only when configured)
- `modules` override list (when omitted/empty, Narratio does not pass `--modules`)
- `base_url` (when omitted, Narratio does not pass `--base-url`)
- `model` (when omitted, Narratio does not pass `--model`)
- `transcript_description`
- `config_path`
- `output_schema` (`bare-segments` or `audita-v1`)
- `work_dir_retention` (`always`, `auto`, `never`)
- `total_llm_concurrency` (> 0 when provided)
- `proposal_llm_concurrency` (> 0 when provided)
- `validation_model`
- `validation_llm_concurrency` (> 0 when provided)
- `report` override
Narratio passes only configured optional Audita flags; omitted optional values defer to Audita runtime defaults/config.
Seriatim defaults:
- `pipeline.seriatim` may be omitted
- `binary` defaults to `seriatim`
- `timeout` defaults to `10m`
- `output_schema` defaults to `seriatim-intermediate`
- `coalesce_gap` defaults to `3.0`
- `report` defaults to `true`
When `pipeline.normalize` is omitted, defaults are applied:
- `output_path: transcripts/normalized.json`
@@ -132,7 +317,7 @@ When `pipeline.trim.enabled: true`:
When `pipeline.scriptorium` is present:
- `binary` is required and non-empty
- `binary` defaults to `scriptorium` when omitted
- `config_path` is optional; when provided it must be non-empty
- `timeout` is optional; when provided it must parse as a Go duration
- default `timeout` is `10m`
@@ -281,6 +466,7 @@ Current expected paths for `session_recap`:
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
- if API-key integration is configured, pass env var names only (never raw key values)
- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs
- avoid logging transcript content or rendered prompt content by default
- treat generated artifacts and logs as potentially sensitive session material

View File

@@ -0,0 +1,507 @@
# Workspace Architecture Implementation Plan (Audit)
## 1. Executive Summary
**Complexity assessment:** **heavy**.
This is not a single path-helper refactor. The current codebase has a hybrid session/run model that works for current behavior, but diverges from `docs/development/workspace.md` in foundational places (workspace root shape, manifest responsibilities, stage output placement, and archive symmetry).
Highest-risk areas:
1. Splitting the current single manifest model into durable **session manifest** vs per-invocation **run manifest** without regressing skip/force/resume UX.
2. Migrating path helpers and artifact-store interfaces from session-only roots (`work/{session}`) to campaign-aware roots (`work/{campaign}/{session}`) while preserving existing runs.
3. Introducing run-local stage outputs + immediate promotion while keeping stage tests and archive behavior stable.
4. Avoiding stale downstream skips after forced upstream reruns.
Surprising findings:
1. Code already has campaign/run-aware helpers (`SessionRunWorkDir`, `SessionSpoolAudioDir`) but core session helpers and manifest pathing remain campaign-unaware.
2. Archive recently gained run/session fallback behavior for manifest/promotion sources, which confirms an existing hybrid-layout pressure point.
3. Analyze input resolution is functional but ad hoc and stage-local; there is no centralized artifact registry/resolver.
---
## 2. Current-State Map
## 2.1 Workspace Path Construction
Primary path model:
- [`internal/artifacts/paths.go`](../../internal/artifacts/paths.go)
- `SessionWorkDir(rootDir, sessionID)` => `{root}/work/{session_id}`
- `buildSessionPaths(workspaceRoot, sessionID)` roots all canonical paths under `{root}/work/{session_id}`
- `SessionRunWorkDir(rootDir, campaign, sessionID, runID)` exists, but is not the default session root helper.
Artifact store abstraction:
- [`internal/artifacts/store.go`](../../internal/artifacts/store.go)
- `SessionPaths(sessionID string)` / `EnsureLayout(sessionID string)` are session-id-only (no campaign argument).
- [`internal/artifacts/local.go`](../../internal/artifacts/local.go)
- `EnsureLayout` creates session-level folders under `SessionWorkDir`.
Path normalization helper:
- [`internal/artifacts/resolve.go`](../../internal/artifacts/resolve.go)
- `ResolveSessionLocalPathForRead` accepts absolute/workspace/session-relative values and probes filesystem.
Where campaign-aware/run-aware support exists:
- Local run path helpers: `SessionRunWorkDir`, `SessionSpoolAudioDir`.
- S3 key helpers: [`internal/artifacts/s3_keys.go`](../../internal/artifacts/s3_keys.go) (`campaigns/sessions/runs/current`).
Where session-root assumptions remain `{workspace}/work/{session}`:
- Manifest path computation: [`internal/app/runner.go`](../../internal/app/runner.go) `manifestPathFor`.
- Artifact store layout and most stage `paths := env.ArtifactStore.SessionPaths(sessionID)` calls.
- Many tests hardcode `workspace/work/<session>/...` (examples below).
Manual/ad hoc path construction (not through a single resolver API):
- Common stage patterns: `filepath.Join(paths.TranscriptsDir, "...")`, `filepath.Join(paths.ArtifactsDir, "...")`, etc.
- `prepare` run/work selection: [`internal/stage/prepare.go`](../../internal/stage/prepare.go) `pathsWorkDirForManifest`.
- Analyze input fallback path resolution: `resolveInputPathForRead` in [`internal/stage/analyze.go`](../../internal/stage/analyze.go).
## 2.2 Manifest and Run Identity
Current manifest model:
- [`internal/manifest/manifest.go`](../../internal/manifest/manifest.go) `Manifest` includes both session and run fields:
- `SessionID`, `Campaign`
- `RunID`, `LocalWorkDir`, `LocalSpoolDir`
- `S3Bucket`, `S3SessionPrefix`, `S3RunPrefix`
- `Stages`, `Inputs`, stage outputs/logs/generated configs/metadata.
Current persistence:
- [`internal/manifest/store.go`](../../internal/manifest/store.go) `LocalStore` reads/writes one JSON manifest path.
- Runner always loads/saves one manifest path via `manifestPathFor(cfg)` (session-root path under current layout).
Identity initialization:
- [`internal/app/runner.go`](../../internal/app/runner.go) `ensureManifestIdentity` populates run fields if absent.
- `RunID` is generated once for an empty manifest and reused thereafter (hybrid semantics).
Interpretation today:
- Best described as a **hybrid session manifest** with run identity fields, not as distinct session + run manifests.
What this means for redesign:
- Session-vs-run split is not just file relocation; it requires new responsibilities and write flows.
- A backward-compatible evolution path is possible by:
- preserving current fields in session manifest for migration/read-compat,
- adding explicit run-manifest type + path,
- gradually moving invocation-specific details to run manifests.
## 2.3 Stage Output Paths (Current)
All implemented stages currently write canonical artifacts directly into session-level `paths.*` roots (under current session root), with logs/configs typically also session-level.
1. `prepare` ([`internal/stage/prepare.go`](../../internal/stage/prepare.go))
- Inputs copied to `inputs/` (`session.yml`, `pipeline.resolved.yml`, `speakers.yml`, `autocorrect.yml`, `glossary.yml`)
- Audio copied to `audio/`
- Manifest input provenance recorded in `m.Inputs`
- S3 audio uses run-scoped spool/work helpers when `RunID` is present.
2. `transcribe` ([`internal/stage/transcribe.go`](../../internal/stage/transcribe.go))
- Outputs: `transcripts/raw/{speaker}.json`
- Stage metadata only (no stage logs/config files produced here).
3. `merge` ([`internal/stage/merge.go`](../../internal/stage/merge.go))
- Pre-normalize intermediates: `transcripts/raw/normalized/{basename}.normalized.json`
- Merge output: `transcripts/merged.json`
- Report: `artifacts/seriatim.report.json` (if enabled)
- Logs/configs:
- per-input normalize logs/configs in session-level `logs/` + `config/`
- merge logs/config in session-level `logs/` + `config/`
4. `polish` ([`internal/stage/polish.go`](../../internal/stage/polish.go))
- Output: `transcripts/processed.json`
- Report: `artifacts/audita.report.json` (if enabled)
- Work dir: `artifacts/audita-work`
- Logs/config: `logs/audita.*`, `config/audita.generated.yml`
5. `normalize` ([`internal/stage/normalize.go`](../../internal/stage/normalize.go))
- Output: `transcripts/normalized.json` (configurable)
- Report: `artifacts/seriatim.normalize.report.json` (if enabled)
- Logs/config: `logs/seriatim.normalize.*`, `config/seriatim.normalize.generated.yml`
6. `trim` ([`internal/stage/trim.go`](../../internal/stage/trim.go))
- Output: `transcripts/trimmed.json` (configurable)
- Bounds output path from config (default examples use `artifacts/session_bounds.json`)
- Logs/configs in session-level `logs/` and `config/` for scriptorium + seriatim invocations
7. `analyze` ([`internal/stage/analyze.go`](../../internal/stage/analyze.go))
- Current implemented artifact: `artifacts/session_recap.md`
- Logs/config in session-level `logs/` + `config/`
- Optional render diagnostics under session-level artifacts/logs/config.
8. `archive` ([`internal/stage/archive.go`](../../internal/stage/archive.go))
- Reads from a "workDir" derived by `archiveWorkDir`:
- prefers `m.LocalWorkDir` if it exists,
- else tries run-scoped campaign/session/run path,
- else falls back to legacy session path.
- Uploads run record and promotions.
- Current code now resolves manifest and promotion sources via run/session fallback.
9. `notify`
- Placeholder only in [`internal/stage/placeholders.go`](../../internal/stage/placeholders.go); no durable outputs.
Durable-vs-diagnostic split today:
- Durable artifacts and diagnostics are mixed at session level.
- No run-local stage directories exist yet.
## 2.4 Idempotency / Force / Resume / Sparse Runs
Run control implementation:
- [`internal/app/run_control.go`](../../internal/app/run_control.go)
- skip rule: `!force && stageSucceeded(manifest, stage)`
- stale detection TODO only; no invalidation logic.
Command behavior:
- `run`: full plan through `executeStages`.
- `run-stage`: single selected stage through `executeStages`.
- `resume`: starts at first non-succeeded stage unless `--force`.
Current assumptions:
- Command invocation mutates a single durable workspace + single manifest for the session path model.
- There is no per-invocation run manifest write path.
Smallest safe UX-preserving invariant to keep during migration:
- Session manifest remains source-of-truth for skip decisions across invocations.
## 2.5 Archive and S3 Alignment
S3 semantics are relatively mature:
- [`internal/artifacts/s3_keys.go`](../../internal/artifacts/s3_keys.go)
- session prefix + run prefix + `current/manifest.json` + `current/run_id.txt`.
Archive stage behavior:
- uploads run records under run prefix,
- uploads promoted session outputs,
- uploads current manifest then current run pointer,
- current pointer is commit marker,
- required promotions fail; optional promotions skipped.
Local-vs-remote mismatch still present:
- Local canonical session root currently defaults to `work/{session}` (artifact store),
- while archive/run helpers expect campaign-aware run locations (`work/{campaign}/{session}/{run}`),
- causing fallback logic and hybrid handling in `archive`.
## 2.6 Artifact Source Resolution in Analyze
Current implementation is stage-local and ad hoc:
- Transcript discoverers:
- `discoverProcessedTranscript`
- `discoverNormalizedTranscript`
- `discoverTrimmedTranscript`
- Input source switch in `resolveScriptoriumInput` supports:
- `processed_transcript`
- `normalized_transcript`
- `trimmed_transcript`
- `previous_session_artifact`
No first-class artifact registry exists yet. Aliases and canonical IDs are not modeled.
## 2.7 Stale/Invalidation
- `manifest.StageStatus` already defines `stale` (`internal/manifest/status.go`), but no stage uses it.
- Skip logic ignores stale state and only checks `succeeded`.
- Forced upstream rerun does not invalidate downstream stage success markers.
## 2.8 Test Coverage Relevant to Redesign
High-value existing coverage:
- Workspace helpers:
- [`internal/artifacts/paths_model_test.go`](../../internal/artifacts/paths_model_test.go)
- [`internal/artifacts/resolve_test.go`](../../internal/artifacts/resolve_test.go)
- Runner semantics:
- [`internal/app/runner_test.go`](../../internal/app/runner_test.go)
- [`internal/app/resume_run_stage_test.go`](../../internal/app/resume_run_stage_test.go)
- Stage path/output behavior:
- `internal/stage/*_test.go` for prepare/transcribe/merge/polish/normalize/trim/analyze/archive
- Archive behavior:
- [`internal/stage/archive_test.go`](../../internal/stage/archive_test.go)
- [`internal/app/post_archive_cleanup_test.go`](../../internal/app/post_archive_cleanup_test.go)
Tests likely to fail during workspace redesign:
- Any tests hardcoding `work/{session}` manifest and transcript paths (many in `internal/app/*test.go`, `internal/stage/*test.go`).
- Archive tests assuming current hybrid fallback behavior.
---
## 3. Gap Analysis Against `docs/development/workspace.md`
| Intended concept | Current status | Notes |
|---|---|---|
| Session root at `work/{campaign}/{session}` | **Partial / mostly absent** | `SessionWorkDir` and artifact store still use `work/{session}`. Campaign-aware run helper exists separately. |
| Distinct session manifest vs run manifest | **Absent** | One hybrid manifest model/file is used. |
| Run-local stage dirs under `runs/{run_id}/{stage}` | **Absent** | Stages write canonical outputs/logs/config directly at session level. |
| Immediate promotion run-local -> session canonical | **Absent** | No run-local staging area to promote from today. |
| Session manifest skip source of truth | **Present** | Skip/resume use single manifest stage statuses. |
| Sparse runs represented under `runs/{run_id}` | **Absent** | No run-manifest/per-run stage records yet. |
| Artifact resolver with canonical IDs + aliases | **Absent** | Analyze resolves via stage-local source-name switch and fallback helpers. |
| Downstream invalidation for forced upstream reruns | **Absent** | TODO only; no stale propagation or status clearing. |
| Local semantics mirror archive semantics | **Partial** | Archive/S3 side models runs/current, local workspace core still session-layout-centric. |
| Safe path helpers centralization | **Partial** | Good helper base exists, but many stage-level manual joins still encode conventions. |
---
## 4. Recommended Implementation Sequence
## Step 1: Introduce campaign-aware session path model without behavior break
Purpose:
- Add first-class helpers for `work/{campaign}/{session}` and make them available everywhere.
Expected changes:
- `internal/artifacts`: add/extend path helpers and `SessionPaths` constructor variants that accept campaign.
- `internal/app`: pass campaign into path-model entrypoints where available.
Behavior change:
- None initially (can keep legacy fallback reads).
Tests:
- Add campaign-aware path-model tests.
- Keep legacy-path compatibility tests.
Risks:
- Wide compile-time touch due `Store` interface signatures.
Rollback:
- Keep legacy helper wrappers until full migration lands.
## Step 2: Split manifest responsibilities (session manifest + run manifest scaffolding)
Purpose:
- Preserve current UX while introducing explicit run execution records.
Expected changes:
- `internal/manifest`: add run manifest type/store helpers.
- `internal/app/runner.go`: create/load session manifest and initialize per-invocation run manifest path.
Behavior change:
- Session manifest remains skip truth source.
- Run manifest begins recording invocation metadata/stage actions.
Tests:
- New tests for both manifest files existing and being updated correctly.
Risks:
- Incorrect ordering of saves can regress crash consistency.
Rollback:
- Keep session-manifest-only decision logic until run manifest proves stable.
## Step 3: Move stage execution products to run-local directories with promotion
Purpose:
- Align with workspace architecture (`runs/{run_id}/{stage}/...`) while preserving canonical outputs.
Expected changes:
- `internal/stage`: each implemented stage writes outputs/logs/config/reports to run-local paths.
- Introduce shared promotion helpers (atomic copy/rename + output validation + manifest provenance).
Behavior change:
- Canonical outputs remain session-level; run-local diagnostics now preserved per run.
Tests:
- Stage tests updated to assert run-local outputs + promoted canonical outputs.
- New tests for immediate promotion and producer run metadata.
Risks:
- Highest regression risk (all implemented stages touched).
Rollback:
- Stage-by-stage migration flag or phased rollout by stage order.
## Step 4: Archive alignment pass
Purpose:
- Remove hybrid fallback complexity once local layout is canonical.
Expected changes:
- `internal/stage/archive.go`: resolve sources from canonical session outputs and run manifests deterministically.
- Keep `current/manifest.json` + `current/run_id.txt` publication semantics.
Behavior change:
- Simpler source selection; fewer cross-layout heuristics.
Tests:
- Archive tests for run uploads, promotions, current pointers, and fallback removal/compat gates.
Risks:
- Breaking current mixed-layout compatibility too early.
Rollback:
- Retain fallback compatibility for one migration window.
## Step 5: Artifact registry/resolver (analyze first consumer)
Purpose:
- Replace ad hoc analyze source resolution with canonical artifact IDs and aliases.
Expected changes:
- New resolver package (for example `internal/artifacts/registry` or `internal/stage/artifactresolve`) with IDs:
- `narratio.transcript.merged`
- `narratio.transcript.polished`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
- `narratio.bounds.session`
- `narratio.artifact.session_recap`
- Backward-compatible alias map for current source names.
Behavior change:
- Analyze input resolution becomes centralized and consistent.
Tests:
- Resolver unit tests for canonical IDs + aliases + missing-input error clarity.
- Analyze tests updated to assert resolver usage.
Risks:
- Input resolution edge cases for `previous_session_artifact` and required/optional handling.
Rollback:
- Keep old resolver path behind a temporary compatibility function.
## Step 6: Minimal downstream invalidation after forced upstream reruns
Purpose:
- Prevent stale downstream skips before full checksum stale detection exists.
Expected changes:
- `internal/app/run_control.go` + manifest transition helpers.
- On forced rerun success of stage `X`, clear/mark downstream success states.
Behavior change:
- Subsequent runs no longer skip stale downstream stages.
Tests:
- New run-control tests for force-induced downstream invalidation.
- Resume tests with forced sparse runs.
Risks:
- Over-invalidating too broadly and degrading UX.
Rollback:
- Start with deterministic downstream stage list based on pipeline order only.
## Step 7: Legacy layout migration strategy
Purpose:
- Handle existing `work/{session}` data safely.
Expected changes:
- Startup detection/migration path in app layer.
- Clear failure messages for ambiguous legacy states.
Behavior change:
- Explicit migration semantics instead of implicit fallback drift.
Tests:
- Migration detection tests for legacy-only, new-only, and ambiguous layouts.
Risks:
- Silent duplication if both layouts are partially populated.
Rollback:
- Prefer fail-fast ambiguity policy over auto-merge.
---
## 5. Minimal Viable v1.0 Scope
Must-have for v1.0:
1. Campaign-aware canonical session roots.
2. Session manifest remains skip/resume authority.
3. Introduce run manifests + run-local stage records.
4. Immediate promotion from run-local outputs to canonical session outputs.
5. Minimal downstream invalidation on forced upstream rerun.
6. Archive/local semantic alignment with `current/*` behavior preserved.
7. Backward-compatible analyze aliases (`processed_transcript`, `normalized_transcript`, `trimmed_transcript`).
Nice-to-have / defer if risky:
1. Full checksum-based stale detection graph.
2. Broad manifest schema-version migration framework.
3. Full artifact-registry rollout beyond analyzes initial needs.
4. Aggressive cleanup of all legacy fallback branches in one release.
---
## 6. Open Questions (with Recommendations)
1. **Should campaign be mandatory for all local path derivation immediately?**
- Recommendation: yes for new layout writes; keep controlled read compatibility for legacy session-only roots during migration window.
2. **Session manifest location transition policy:** auto-migrate vs explicit migrate command?
- Recommendation: if user base is small, prefer explicit fail-fast with actionable migration instructions to avoid silent split-brain state.
3. **Run manifest granularity:** per-stage detailed records vs summary + references?
- Recommendation: start with summary + stage status/paths; avoid duplicating full session artifact state to keep write path simple.
4. **Invalidation marking:** use `stale` status now or clear `succeeded` markers?
- Recommendation: if invasive to propagate new status semantics quickly, clear/overwrite downstream success states first; add formal `stale` usage in follow-up.
5. **Promotion timing:** promote every stage immediately vs delayed at end of run?
- Recommendation: immediate per-stage promotion after validation (matches current idempotent skip expectations and simplifies resume behavior).
6. **Docs consistency order after implementation starts:**
- Recommendation: update `README.md` and `architecture.md` in lockstep with each migration step. Current conflict is explicit:
- code/README/architecture still primarily describe session roots at `work/{session}`
- `docs/development/workspace.md` defines `work/{campaign}/{session}` plus run manifests.

View File

@@ -0,0 +1,753 @@
# Narratio Workspace, Run History, and Artifact Resolution Architecture
## 1. Purpose
This document defines the intended v1.0 architecture for Narratio's local workspace layout, run history model, durable session outputs, manifest responsibilities, and artifact resolution contract.
Narratio is an idempotent session orchestrator. The command:
```bash
narratio run --session-id 2026-05-07
```
means "bring the identified session to its desired completed state." It does **not** mean "always create an entirely new independent output tree and ignore prior session state."
This distinction drives the architecture:
* A **session** is the durable domain object and idempotency boundary.
* A **run** is an execution attempt that may update the session's durable state.
* Durable outputs live at the session level.
* Run-specific outputs, logs, generated configs, scratch files, and diagnostics live under `runs/{run_id}/`.
* Successful stage outputs are promoted from run-local locations into canonical session-level locations.
* The session manifest records current durable state.
* Run manifests record execution history and debugging/provenance details.
This model intentionally mirrors the S3 archive model: session-level current artifacts are distinct from run-record history.
## 2. Core Concepts
### 2.1 Session
A session is the stable unit of work identified by `campaign_id` and `session_id`.
Examples:
```text
campaign_id = dilfs
session_id = 2026-05-07
```
The session directory represents the current durable local state for that session. Re-running Narratio for the same session should consult this state, skip already-completed stages by default, and produce no changes unless work is incomplete, stale, forced, or explicitly selected.
### 2.2 Run
A run is a particular execution attempt identified by a generated `run_id`, for example:
```text
20260517T174748Z-abcd1234
```
A run may execute all stages or only a sparse subset of stages. Sparse runs are expected and desirable when the user invokes `--force`, `run-stage`, or a stage-limited command.
Run directories are provenance/debug records. They should reflect what actually happened during that invocation, not a synthetic complete pipeline layout.
### 2.3 Durable Output
A durable output is a canonical session-level artifact intended for later stages, user consumption, archive promotion, or future idempotency decisions.
Examples:
```text
transcripts/merged.json
transcripts/processed.json
transcripts/normalized.json
transcripts/trimmed.json
artifacts/session_recap.md
```
Durable outputs live directly under the session directory, not under a particular run directory.
### 2.4 Run-Local Output
A run-local output is the file initially produced by a stage during a specific run. After validation, durable outputs are promoted from run-local paths to session-level canonical paths.
Run-local outputs, logs, generated configs, reports, and scratch files should remain under:
```text
runs/{run_id}/{stage}/...
```
## 3. Local Workspace Layout
The canonical local workspace layout is:
```text
{workspace.root}/work/{campaign_id}/{session_id}/
manifest.json
current/
manifest.json
run_id.txt
inputs/
transcripts/
artifacts/
reports/
logs/
config/
runs/
{run_id}/
manifest.json
prepare/
transcribe/
merge/
polish/
normalize/
trim/
analyze/
archive/
notify/
```
Not every directory must exist at all times. Directories should be created idempotently when needed.
### 3.1 Session Root
The session root is:
```text
{workspace.root}/work/{campaign_id}/{session_id}/
```
The session root is the stable local home for the session. It is the default base for resolving canonical artifact paths.
The only files that should live directly in the session root are core session-state files, primarily:
```text
manifest.json
```
Lock files may also be session-root scoped if the implementation uses file locks there, but transient locks should not be treated as durable artifacts.
### 3.2 Session-Level Canonical Directories
The following directories contain current durable session state:
```text
inputs/
transcripts/
artifacts/
reports/
logs/
config/
current/
```
Recommended meanings:
| Directory | Purpose |
| -------------- | ----------------------------------------------------------------------------- |
| `inputs/` | Materialized or copied input files used by the current durable session state. |
| `transcripts/` | Canonical transcript tiers. |
| `artifacts/` | User-facing and machine-readable generated artifacts. |
| `reports/` | Canonical stage reports worth preserving at the session level. |
| `logs/` | Optional session-level logs or promoted/latest logs. |
| `config/` | Optional session-level generated config snapshots or promoted/latest configs. |
| `current/` | Current published session pointers, mirroring the archive backend. |
Canonical durable outputs should use stable paths under these directories.
### 3.3 Run History Directory
Run history lives under:
```text
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/
```
Each run directory records what happened during that invocation. A run may contain all stage directories or only a sparse subset.
Example full run:
```text
runs/20260517T174748Z-abcd1234/
manifest.json
prepare/
transcribe/
merge/
polish/
normalize/
trim/
analyze/
archive/
notify/
```
Example sparse forced analyze run:
```text
runs/20260518T030000Z-efgh5678/
manifest.json
analyze/
```
Example sparse polish-through-analyze rerun:
```text
runs/20260518T041500Z-a1b2c3d4/
manifest.json
polish/
normalize/
trim/
analyze/
```
Run directories should not create stage folders for stages that were not selected, executed, skipped, or otherwise considered during that run unless there is a clear diagnostic reason to do so.
### 3.4 Stage Run-Local Directories
Each stage receives a run-local directory:
```text
runs/{run_id}/{stage}/
```
Within that stage directory, the stage may use subdirectories such as:
```text
outputs/
logs/
reports/
config/
scratch/
```
For example:
```text
runs/{run_id}/polish/
outputs/transcripts/processed.json
reports/audita.polish.report.json
logs/stdout.log
logs/stderr.log
config/audita.polish.generated.yml
scratch/
```
The exact internal layout of a stage directory may vary by stage, but it should be deterministic, documented, and generated through centralized path helpers rather than ad hoc path joins.
## 4. Promotion Model
Narratio uses stage-level promotion with immediate promotion after successful validation.
The stage lifecycle is:
1. Resolve required inputs from the current session state and/or run-local context.
2. Create the run-local stage directory.
3. Execute the stage, writing outputs under `runs/{run_id}/{stage}/...`.
4. Validate run-local outputs.
5. Promote durable outputs into session-level canonical paths.
6. Update the session manifest.
7. Update the run manifest.
Promotion means an atomic or effectively atomic copy/rename from a run-local path to a session-level canonical path.
Example:
```text
runs/{run_id}/polish/outputs/transcripts/processed.json
```
is promoted to:
```text
transcripts/processed.json
```
Promotion should be safe and deterministic:
* Validate before promotion.
* Write promoted files atomically where possible.
* Never leave partially written durable outputs.
* Record the producing `run_id` in the session manifest.
* Preserve run-local files for debugging unless retention policy deletes them.
## 5. Promotion Policy: Option A
Narratio uses immediate stage-level promotion.
If a selected stage succeeds, its durable outputs are promoted immediately, even if a later selected stage fails.
Example:
```bash
narratio run --session-id 2026-05-07 --force --stages polish,normalize,trim,analyze
```
If `polish` succeeds and `normalize` fails:
* `transcripts/processed.json` may be updated from the new run.
* `normalize`, `trim`, and `analyze` should not be marked succeeded for the new input state.
* Downstream outputs may now be stale relative to the newly promoted polished transcript.
This policy is simpler, transparent, and consistent with stage-level resumability. It does require explicit stale/invalidation handling.
## 6. Stale and Invalidation Semantics
Full checksum-based stale detection may be implemented later. Before that exists, Narratio should still use a simple deterministic invalidation rule for forced or explicit upstream reruns.
When a stage is successfully re-executed and promoted, downstream stages should be marked stale unless they are also re-executed successfully in the same command invocation.
Example stage order:
```text
prepare -> transcribe -> merge -> polish -> normalize -> trim -> analyze -> archive -> notify
```
If `polish` is forced and promoted, then the following downstream stages should be invalidated unless rerun successfully:
```text
normalize
trim
analyze
archive
notify
```
A stale stage is not equivalent to a failed stage. It means its current durable outputs may no longer correspond to current upstream inputs or configuration.
Minimum manifest state model:
```text
pending
running
succeeded
failed
skipped
stale
```
If adding a new `stale` state is too invasive for v1.0, the implementation should at least record stale metadata or clear downstream success markers in a way that prevents accidental idempotent skips based on obsolete outputs.
## 7. Manifest Responsibilities
Narratio should distinguish between session manifests and run manifests.
The same underlying Go types may be reused where practical, but the concepts should remain separate.
### 7.1 Session Manifest
Path:
```text
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
```
The session manifest answers:
```text
What is the current durable state of this session?
```
It should record:
* campaign ID
* session ID
* current or latest run ID
* current stage states
* canonical durable output refs
* artifact IDs and paths
* producing run ID for each current stage output
* relevant input/config checksums when available
* stale/invalidated stage information
* archive/current publication metadata
A session's durable state may be a composite of multiple runs.
For example:
```text
transcripts/merged.json produced by run A
transcripts/processed.json produced by run B
transcripts/normalized.json produced by run B
transcripts/trimmed.json produced by run B
artifacts/session_recap.md produced by run C
```
This is valid and expected.
### 7.2 Run Manifest
Path:
```text
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/manifest.json
```
The run manifest answers:
```text
What happened during this specific execution attempt?
```
It should record:
* run ID
* campaign ID
* session ID
* command mode and selected stages
* force flags or stage selection flags
* stages considered during this run
* stages executed during this run
* stages skipped during this run and reasons
* run-local output paths
* promoted output paths
* logs
* reports
* generated configs
* timings
* errors
* non-secret subprocess invocation metadata
Run manifests are primarily for debugging, auditability, and archive history.
## 8. Idempotency and Resume Behavior
The idempotency boundary is the session, not the run.
By default:
```bash
narratio run --session-id 2026-05-07
```
should consult the session manifest and skip stages that are already succeeded and not stale.
If all stages are already complete, the command should execute zero stages and report that the session is already complete.
Forced execution creates a new run record but updates session-level durable state only for stages that actually succeed and promote outputs.
Examples:
```bash
narratio run --session-id 2026-05-07 --force
```
Creates a new run and attempts to re-execute the selected/default stage set.
```bash
narratio run-stage --session-id 2026-05-07 analyze --force
```
Creates a sparse run that executes only `analyze`, then promotes updated analysis artifacts if successful.
```bash
narratio resume --session-id 2026-05-07
```
Uses the session manifest to determine what remains incomplete or stale. Resume does not need to resume the same `run_id` unless the implementation explicitly supports resuming an interrupted active run.
## 9. Artifact Resolution Contract
Narratio should provide a first-class artifact registry and resolver.
The resolver maps symbolic artifact source names to canonical session-level paths and manifest output kinds.
Stages and adapters should not hardcode path fragments when resolving cross-stage inputs. They should ask the artifact resolver for the current durable artifact by ID.
### 9.1 Canonical Artifact IDs
Preferred artifact IDs should be namespaced:
```text
narratio.transcript.merged
narratio.transcript.polished
narratio.transcript.full
narratio.transcript.trimmed
narratio.bounds.session
narratio.artifact.session_recap
```
Recommended initial registry:
| Artifact ID | Canonical Path | Producer Stage | Output Kind | Meaning |
| --------------------------------- | ------------------------------- | -------------- | ------------------------ | ------------------------------------- |
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` | Deterministic Seriatim merge. |
| `narratio.transcript.polished` | `transcripts/processed.json` | `polish` | `transcript_processed` | Full Audita-polished transcript. |
| `narratio.transcript.full` | `transcripts/normalized.json` | `normalize` | `transcript_normalized` | Preferred full normalized transcript. |
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` | Gameplay-only transcript. |
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` | Trim bounds selected for the session. |
| `narratio.artifact.session_recap` | `artifacts/session_recap.md` | `analyze` | `artifact_session_recap` | Generated session recap. |
### 9.2 Backward-Compatible Aliases
Existing source names should remain supported:
| Legacy Source | Preferred Artifact ID |
| ----------------------- | ------------------------------ |
| `processed_transcript` | `narratio.transcript.polished` |
| `normalized_transcript` | `narratio.transcript.full` |
| `trimmed_transcript` | `narratio.transcript.trimmed` |
These aliases may be supported silently for v1.0. Documentation should prefer namespaced IDs.
### 9.3 Resolver Behavior
Artifact resolution should follow this order:
1. Normalize aliases to canonical artifact IDs.
2. Look for a current output reference in the session manifest.
3. Fall back to the canonical session-level path.
4. If the artifact is required, fail clearly if missing.
5. If the artifact is optional and missing, omit it from the downstream invocation.
6. Validate the artifact using the expected content validator.
7. Return a resolved artifact record containing ID, path, producer stage, output kind, and provenance.
Example conceptual result:
```json
{
"id": "narratio.transcript.trimmed",
"path": "/var/lib/narratio/work/dilfs/2026-05-07/transcripts/trimmed.json",
"producer_stage": "trim",
"producer_run_id": "20260517T174748Z-abcd1234",
"output_kind": "transcript_trimmed",
"content_type": "application/json"
}
```
### 9.4 Artifact Validation
Transcript artifacts must be valid JSON with a top-level `segments` array.
Markdown/text artifacts must exist and be non-empty when required.
Bounds artifacts must match the expected bounds schema and refer to segment IDs in the same transcript ID space used by the trim stage.
Validation should happen before a resolved artifact is passed to another stage or external subprocess.
## 10. Analyze Stage Implications
The analyze stage should consume artifacts through the artifact resolver.
Preferred Scriptorium config shape:
```yaml
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: "dnd.session_recap"
output_path: "artifacts/session_recap.md"
inputs:
transcript:
source: "narratio.transcript.trimmed"
required: true
```
Additional artifacts can choose different transcript tiers:
```yaml
scriptorium:
artifacts:
table_summary:
enabled: true
prompt_id: "dnd.table_summary"
output_path: "artifacts/table_summary.md"
inputs:
transcript:
source: "narratio.transcript.full"
required: true
```
For v1.0, Narratio does not need a generic DAG engine. It may execute configured analyze artifacts in deterministic order and allow later artifacts to consume earlier artifacts only when that relationship is explicit and unambiguous.
Rules:
* Artifact inputs resolve from current session-level durable state.
* Outputs are first written run-locally.
* Successful analyze outputs are promoted to session-level `artifacts/` paths.
* Manifest output refs record the producing run ID.
* Optional inputs are omitted when unavailable.
* Required missing inputs fail before invoking Scriptorium.
## 11. Archive Alignment
Local workspace semantics should mirror archive semantics.
Local session-level durable paths:
```text
work/{campaign}/{session}/transcripts/trimmed.json
work/{campaign}/{session}/artifacts/session_recap.md
work/{campaign}/{session}/current/manifest.json
work/{campaign}/{session}/current/run_id.txt
work/{campaign}/{session}/runs/{run_id}/...
```
should map naturally to remote archive paths:
```text
{root_prefix}/campaigns/{campaign}/sessions/{session}/transcripts/trimmed.json
{root_prefix}/campaigns/{campaign}/sessions/{session}/artifacts/session_recap.md
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/manifest.json
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/run_id.txt
{root_prefix}/campaigns/{campaign}/sessions/{session}/runs/{run_id}/...
```
The archive stage should publish run records and promoted current artifacts consistently with the local model.
`current/run_id.txt` remains the effective commit marker for the archived current session state.
## 12. Path Helper Requirements
All code should use centralized path helpers for workspace paths.
Stage code should not manually assemble durable cross-stage paths using raw string joins except through the path model.
Recommended helper surface:
```text
SessionRoot(campaignID, sessionID)
SessionManifestPath(campaignID, sessionID)
SessionCurrentDir(campaignID, sessionID)
SessionTranscriptsDir(campaignID, sessionID)
SessionArtifactsDir(campaignID, sessionID)
SessionReportsDir(campaignID, sessionID)
SessionLogsDir(campaignID, sessionID)
SessionConfigDir(campaignID, sessionID)
RunsDir(campaignID, sessionID)
RunRoot(campaignID, sessionID, runID)
RunManifestPath(campaignID, sessionID, runID)
RunStageDir(campaignID, sessionID, runID, stage)
RunStageOutputsDir(campaignID, sessionID, runID, stage)
RunStageLogsDir(campaignID, sessionID, runID, stage)
RunStageReportsDir(campaignID, sessionID, runID, stage)
RunStageConfigDir(campaignID, sessionID, runID, stage)
CanonicalArtifactPath(campaignID, sessionID, artifactID)
```
Path helpers should enforce safe relative paths for configured output paths:
* reject absolute paths unless explicitly allowed for a particular config field
* reject `..` traversal
* normalize separators
* preserve deterministic output paths
## 13. Directory Creation Policy
Directory creation should be centralized and idempotent.
Recommended policy:
* `prepare` ensures the baseline session directory structure exists.
* Every stage also calls shared layout helpers to ensure its required run-local directories exist before writing.
* `run-stage` should not depend on a prior `prepare` invocation merely to create folders.
* Missing directories should be created with appropriate permissions.
* Directory creation should not imply stage success.
This provides consistent layout while keeping direct stage execution robust.
## 14. Cleanup and Retention
Cleanup must preserve the distinction between durable session state and run history.
Workspace cleanup after successful archive may remove selected local directories only according to explicit configuration.
Potential retention policies:
```text
keep_all_runs
keep_failed_runs
keep_last_n_runs
delete_run_after_success
```
For v1.0, conservative retention is preferred:
* Do not delete durable session-level outputs unless explicitly requested.
* Do not delete failed run directories by default.
* If cleanup is enabled, remove only documented run-scoped or spool-scoped paths.
* Local development audio inputs must never be deleted by workspace cleanup.
## 15. Migration From Existing Layout
Existing installations may currently use a simpler path such as:
```text
{workspace.root}/work/{session_id}/manifest.json
```
The v1.0 layout introduces campaign-aware session roots:
```text
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
```
Migration options:
1. Best-effort automatic discovery of legacy session manifests.
2. A one-time migration command.
3. Clear release notes requiring users to move or regenerate workspace state.
For v1.0, it is acceptable to require explicit migration if the user base is small and the archive contains the authoritative durable outputs. However, the application should fail clearly when it detects an ambiguous legacy layout rather than silently creating duplicate state.
## 16. Documentation Updates Required
The following documentation should be updated to reflect this architecture:
* `README.md`
* `docs/architecture.md`
* a dedicated workspace/run-history document, such as this file
* S3/archive documentation
* analyze/artifact configuration documentation
* example pipeline files
Documentation should consistently use the following terms:
| Term | Meaning |
| ---------------- | ---------------------------------------------------------------------- |
| Session | Durable domain object and idempotency boundary. |
| Run | Execution attempt that may update session state. |
| Durable output | Canonical current session-level output. |
| Run-local output | Output produced inside a specific run directory before promotion. |
| Promotion | Validated copy/rename from run-local output to durable session output. |
| Session manifest | Current durable state of the session. |
| Run manifest | Execution record for a particular run. |
| Artifact ID | Symbolic source name resolved by the artifact registry. |
## 17. Architectural Invariants
The following invariants should hold after implementation:
1. `session_id` remains the idempotency boundary for normal operator commands.
2. `run_id` identifies an execution attempt, not the primary durable workspace.
3. Session-level canonical artifacts are the default inputs for downstream stages.
4. Run-local outputs are promoted only after validation.
5. A session's current durable state may be composed of outputs from multiple runs.
6. Sparse run directories are valid and expected.
7. The session manifest records current stage/artifact state and producer run IDs.
8. The run manifest records what happened during one invocation.
9. Artifact consumers resolve symbolic artifact IDs through a registry/resolver.
10. Local workspace semantics mirror S3 archive semantics.
11. Directory creation is centralized and idempotent.
12. Stage code uses path helpers rather than ad hoc path construction.
13. Forced upstream reruns invalidate downstream stage success unless downstream stages are rerun successfully.
14. Cleanup never removes durable session outputs or local development inputs unless explicitly configured to do so.
## 18. Implementation Guidance
A practical implementation sequence is:
1. Add this architecture document.
2. Add or revise path model helpers for session roots, run roots, stage directories, and canonical artifact paths.
3. Introduce session manifest versus run manifest concepts.
4. Route stage outputs through run-local directories.
5. Add promotion helpers with validation and atomic writes.
6. Update existing stages to promote durable outputs to session-level canonical paths.
7. Add artifact registry and resolver.
8. Update analyze to use artifact IDs and aliases.
9. Add simple downstream stale invalidation for forced upstream reruns.
10. Align archive/local path behavior and documentation.
11. Update examples and README.
12. Add tests for idempotency, sparse forced runs, promotion, manifest provenance, and artifact resolution.
This sequence intentionally avoids introducing a generic DAG engine. The v1.0 goal is a clear, deterministic, stage-oriented orchestrator with stable session-level outputs and inspectable run history.

View File

@@ -1,147 +1,96 @@
# Audita
# Audita Subprocess Operations
Audita is a framework-first transcript correction application. The public `audita` package provides:
This document describes how parent processes should invoke `audita process` safely in production orchestration.
- deterministic transcript normalization
- token-batched module orchestration
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
- structured run reporting and work-dir diagnostics
## Recommended command form
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
## Development
This project is set up for `uv`.
Use explicit file outputs for orchestrated runs:
```sh
uv sync --extra dev
uv run pytest
audita process <transcript.json> \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--glossary <glossary.yaml> \
--output <output-transcript.json> \
--report-json <report.json>
```
## Usage
Additional flags that may be situationally appropriate:
- `--config <path>` to select an explicit versioned config file.
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
- `--work-dir <dir>` to control diagnostics location.
- `--work-dir-retention <always|auto|never>` to control retained run directories.
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
- `--modules ...` only when intentionally overriding the default sequence.
Process a transcript with the current framework implementation:
For config-driven orchestration, validate config files in CI/preflight:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
audita config validate --config <path>
```
The framework currently runs this default module sequence:
## Stdout behavior
1. `glossary`
2. `homophones`
3. `glossary`
4. `spoken_word`
5. `grammar`
- With `--output`: stdout is expected to be empty on success.
- Without `--output`: stdout contains transcript JSON only on success.
- Report JSON is never written to stdout.
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
## Stderr behavior
1. `glossary_1`
2. `homophones`
3. `glossary_2`
4. `spoken_word`
5. `grammar`
- Success path should be quiet or minimal human-readable logs.
- Failure path writes concise human-readable errors.
- When a diagnostics run directory exists, failure stderr includes its path.
- Prompt/response diagnostic payloads are not streamed to stderr.
The default module sequence is fully implemented today:
## Output file behavior
- `glossary` proposes glossary-supported acoustic corrections
- `homophones` proposes conservative homophone and mistranscription corrections
- `spoken_word` proposes conservative dysfluency cleanup
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
- `--output` writes transcript JSON in the selected output schema to the provided path.
- Output write failures return nonzero and surface actionable errors.
- The command does not silently ignore output write errors.
To run a custom module sequence, pass `--modules`:
## Report JSON behavior
```sh
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
```
- `--report-json` writes a machine-readable process report to the requested path.
- Run-directory `report.json` is written independently under diagnostics.
- Best-effort failure reports are emitted when possible without masking the primary failure.
- Report write failures return nonzero with clear stderr messaging.
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
To also write a structured JSON report:
## Diagnostics directory behavior
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
```
- Each run creates (when possible) a per-run diagnostics directory.
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
- Failed runs retain diagnostics.
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
From a checked-out repository, you can also use the root launcher:
## Exit codes
```sh
./audita process transcript.json --glossary glossary.yaml --output corrected.json
```
- `0`: success.
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
Treat any nonzero as a failed subprocess invocation.
```sh
cd /usr/local/src/audita
uv sync --extra dev
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
## Timeout and cancellation
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
- On cancellation or timeout, the process exits nonzero and should not hang.
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
## Secret redaction expectations
| Environment variable | CLI flag | Default | Purpose |
| --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
API keys and configured secret values are redacted from:
- reports (`--report-json` and run-dir `report.json`);
- diagnostics artifacts (including effective config and LLM interaction artifacts);
- surfaced adapter/runtime errors;
- test fixtures and regression outputs.
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Parent-process logs should still avoid printing raw environment variables.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
## Parent-process pipe guidance
OpenRouter remains the default out of the box:
To avoid deadlocks in orchestrators:
- always read both stdout and stderr concurrently when invoking as a subprocess;
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
- treat stderr as human-readable diagnostics, not structured data;
- parse structured results from output/report files.
```sh
export AUDITA_LLM_API_KEY=your-openrouter-key
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
```sh
export AUDITA_LLM_API_KEY=local-dev-key
export AUDITA_BASE_URL=http://localhost:8000/v1
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
Or the actual OpenAI API:
```sh
export AUDITA_LLM_API_KEY=your-openai-key
export AUDITA_BASE_URL=https://api.openai.com/v1
export AUDITA_MODEL=gpt-4.1-mini
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
## Prototype Archive
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.

119
docs/stages/archive.md Normal file
View File

@@ -0,0 +1,119 @@
# Archive Storage
This document describes implemented archive-stage publish behavior.
## S3 Paths
Session root:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
Run prefix:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
## Scope
Implemented:
- archive uploads successful run records to remote object storage through the storage backend abstraction.
- archive uploads configured promoted outputs to session-level keys.
- archive uploads `current/manifest.json`.
- archive uploads `current/run_id.txt` last as the effective commit marker.
- optional post-archive local cleanup:
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir
- tests use fake storage and do not require live S3.
Future work:
- `notify` stage behavior
- stale detection
- optional future source-audio upload mode
- additional artifact generation beyond current implemented set
## Prerequisites
Archive verifies these stages succeeded before upload:
- `prepare`
- `transcribe`
- `merge`
- `polish`
- `normalize`
- `trim`
- `analyze`
If any prerequisite is missing or not succeeded, archive fails and does not upload.
Failed or incomplete runs remain local only.
## Run Upload
Archive uploads existing files from the run workdir when present:
- `inputs/`
- `transcripts/`
- `artifacts/`
- `reports/` (optional)
- `config/`
- `logs/`
- `manifest.json`
Relative paths are preserved under `runs/{run_id}/`.
## Promotion Rules
Archive applies `archive.promote_artifacts` in config order.
Rule behavior:
- `from`: local workdir-relative source path
- `to`: session-root-relative destination key
- `required: true`: missing source fails archive
- `required: false`: missing source is skipped and recorded
Default promoted outputs:
- `transcripts/trimmed.json`
- `artifacts/session_recap.md`
## Current Pointers
Archive writes:
1. `current/manifest.json` (after run upload + promotions)
2. `current/run_id.txt` last
`current/run_id.txt` contains exactly:
- `{run_id}` plus trailing newline
Writing `current/run_id.txt` last makes it the effective commit marker for published session state.
If any required run upload, promotion upload, or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`.
Cleanup runs only after this commit-marker write has succeeded.
## Audio Upload Policy
Archive does not upload local `audio/` by default.
Original audio is expected at the session-level audio prefix and is not duplicated under `runs/{run_id}/`.
## Config Controls
- `archive.enabled: false` skips archive cleanly.
- `archive.upload_run: false` skips run upload cleanly.
- both skip cases also skip post-archive local cleanup.
## Metadata
Archive stage metadata includes non-secret upload context (for example):
- `s3_bucket`
- `s3_run_prefix`
- run upload counts/paths
- promoted upload counts/paths
- skipped optional promotions
- `current_manifest_key`
- `current_run_id_key`
- `current_pointer_written`
- `audio_upload_skipped`

84
docs/stages/prepare.md Normal file
View File

@@ -0,0 +1,84 @@
# S3 Audio Input
This document describes implemented S3 audio input behavior in `prepare`.
## Scope
Implemented:
- `prepare` can acquire source audio from S3 when `session.inputs.audio_s3.prefix` is configured.
- object listing and download go through the storage backend abstraction.
- tests use fake storage; no live S3 service is required for test runs.
Not implemented:
- uploads of failed runs
## Required Configuration
`pipeline.yml`:
- `storage.s3.bucket` must be set when S3 audio input is used.
- `storage.s3.root_prefix` defaults to `dnd`.
- `storage.s3.access_key_id_env` defaults to `OBJECT_STORAGE_KEY_ID`.
- `storage.s3.secret_access_key_env` defaults to `OBJECT_STORAGE_KEY`.
- `spool.root` defaults to `/var/spool/narratio`.
`session.yml`:
- configure `session.campaign` and `session.session_id`.
- configure `session.inputs.audio_s3.prefix` for S3 audio input.
- do not configure `inputs.audio_dir` or `inputs.audio_files` at the same time as `inputs.audio_s3`.
## Prefix Shape
Session S3 root:
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
Audio prefix:
`{session_root}/{audio_s3.prefix}`
Example:
`dnd/campaigns/forsaken/sessions/2026-04-19/audio/`
Audio files must already exist in S3 before running Narratio.
## Prepare Behavior
When `inputs.audio_s3.prefix` is configured, `prepare`:
1. lists objects under the computed S3 audio prefix
2. filters to `.flac` objects
3. fails when no `.flac` objects are found
4. downloads selected objects to spool audio:
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
5. materializes audio into workdir audio:
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
6. records input provenance in the manifest (bucket, key, metadata, local paths, checksum)
Notes:
- `.flac` filtering is case-insensitive.
- ETag is recorded as provider metadata only and is not treated as a checksum.
## Local Audio Development
Local audio workflows remain supported:
- `inputs.audio_dir`
- `inputs.audio_files`
These options are mutually exclusive with `inputs.audio_s3`.
## Archive Boundary
Current archive behavior relevant to S3 audio input:
- successful runs are uploaded by archive under `runs/{run_id}/`
- configured promotions are uploaded to session-level destinations
- `current/manifest.json` and `current/run_id.txt` are published
- local source audio is not re-uploaded by default
- failed or incomplete runs are not uploaded

View File

@@ -0,0 +1,26 @@
workspace:
root: ./tmp/narratio-workspace
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
seriatim:
binary: "seriatim"
timeout: "10m"
output_schema: "seriatim-intermediate"
coalesce_gap: 3.0
audita:
binary: "audita"
timeout: "3h"
base_url: "https://openrouter.ai/api/v1"
model: "openrouter/google/gemma-4-31b-it"
llm_api_key_env: "AUDITA_LLM_API_KEY"
modules: ["glossary", "homophones", "spoken_word", "grammar"]
output_schema: "audita-v1"
work_dir_retention: "auto"
total_llm_concurrency: 2
proposal_llm_concurrency: 1
validation_model: "openrouter/google/gemma-4-31b-it"
validation_llm_concurrency: 1
report: true

View File

@@ -1,97 +1,51 @@
workspace:
root: ./tmp/narratio-workspace
cleanup_after_archive: false
storage:
backend: local
backend: s3
s3:
bucket: "my-dnd-archive"
root_prefix: "dnd"
region: "us-east-1"
# Optional credential env-var names (defaulted when omitted):
# access_key_id_env: "OBJECT_STORAGE_KEY_ID"
# secret_access_key_env: "OBJECT_STORAGE_KEY"
spool:
root: "/var/spool/narratio"
delete_audio_after_archive: false
archive:
enabled: true
upload_run: true
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
language: "en"
timeout: "30m"
retries: 3
retry_delay: "2s"
concurrency: 2
seriatim:
binary: "seriatim"
timeout: "10m"
output_schema: "seriatim-intermediate"
coalesce_gap: 3.0
report: true
env:
overlap_word_run_gap: 1.0
overlap_word_run_reorder_window: 1.0
backchannel_max_duration: 2.0
filler_max_duration: 1.25
# Optional. When omitted entirely, Narratio defaults to seriatim binary + runtime defaults.
seriatim: {}
# Optional runtime overrides. Model/provider can be owned by Audita runtime config.
audita:
binary: "audita"
timeout: "3h"
config_path: "/usr/local/etc/audita/config.yml"
llm_api_key_env: "AUDITA_LLM_API_KEY"
modules:
- glossary
- homophones
- glossary
- spoken_word
- grammar
- homophones
- glossary
base_url: "https://openrouter.ai/api/v1"
model: "openrouter/google/gemma-4-31b-it"
llm_concurrency: 1
validation_model: ""
validation_llm_concurrency: 1
report: true
normalize:
# Session-workdir-relative when not absolute.
output_path: "transcripts/normalized.json"
output_schema: "seriatim-intermediate"
report: true
trim:
enabled: true
# Session-workdir-relative when not absolute.
output_path: "transcripts/trimmed.json"
bounds:
prompt_id: "dnd_session.bounds"
# Empty means use prompt default profile.
profile_id: ""
transcript_input_name: "transcript"
output_path: "artifacts/session_bounds.json"
timeout: "10m"
render_debug: false
render_output_path: "artifacts/session_bounds.render.json"
seriatim:
report: false
# Optional Scriptorium integration for analyze artifacts.
scriptorium:
binary: "scriptorium"
config_path: "/etc/scriptorium/config.yml"
timeout: "10m"
render_debug: false
config_path: "/usr/local/etc/scriptorium/config.yml"
artifacts:
session_recap:
enabled: true
prompt_id: "dnd.session_recap"
profile_id: "local-quality"
output_path: "artifacts/session_recap.md"
timeout: "10m"
# Optional per-artifact override of global scriptorium.render_debug.
# render_debug: true
inputs:
transcript:
# Available transcript sources:
# - trimmed_transcript (recommended for session_recap)
# - normalized_transcript (recommended for future full-session analysis)
# - processed_transcript (raw Audita-polished output)
source: "trimmed_transcript"
required: true
previous_recap:
source: "previous_session_artifact"
artifact: "session_recap"
# Optional: set when previous recap is available.
path: ""
required: false
vars:
session_id: true
@@ -99,11 +53,3 @@ scriptorium:
campaign_name: true
previous_session_id: true
output_kind: "session_recap"
analyzer:
timeout: 20m
artifacts:
output_dir: artifacts
notification:
timeout: 10s

View File

@@ -4,7 +4,10 @@ date: 2026-05-03
title: Sample Session
inputs:
audio_dir: ./audio
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
# Narratio prepare lists this prefix and downloads .flac files.
# audio_s3:
# prefix: "audio/"
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

View File

@@ -0,0 +1,12 @@
session_id: "{{ session_id }}"
campaign: sample-campaign
date: ""
title: ""
inputs:
audio_dir: ./audio
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
# audio_s3:
# prefix: "audio/{{ session_id }}/"
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml

25
go.mod
View File

@@ -2,4 +2,27 @@ module gitea.maximumdirect.net/eric/narratio
go 1.25.0
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/smithy-go v1.25.1
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
)

36
go.sum
View File

@@ -1,3 +1,39 @@
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -24,6 +24,12 @@ type PolishRequest struct {
Modules []string
BaseURL string
Model string
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
StdoutLogPath string

View File

@@ -21,7 +21,12 @@ type SubprocessRunnerConfig struct {
Modules []string
BaseURL string
Model string
LLMConcurrency *int
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
Report bool
@@ -35,7 +40,12 @@ type SubprocessRunner struct {
modules []string
baseURL string
model string
llmConcurrency *int
transcriptDescription string
configPath string
outputSchema string
workDirRetention string
totalLLMConcurrency *int
proposalLLMConcurrency *int
validationModel string
validationLLMConcurrency *int
report bool
@@ -49,7 +59,12 @@ func NewSubprocessRunnerFromConfigValues(
modules []string,
baseURL string,
model string,
llmConcurrency *int,
transcriptDescription string,
configPath string,
outputSchema string,
workDirRetention string,
totalLLMConcurrency *int,
proposalLLMConcurrency *int,
validationModel string,
validationLLMConcurrency *int,
report bool,
@@ -68,7 +83,12 @@ func NewSubprocessRunnerFromConfigValues(
Modules: modules,
BaseURL: baseURL,
Model: model,
LLMConcurrency: llmConcurrency,
TranscriptDescription: transcriptDescription,
ConfigPath: configPath,
OutputSchema: outputSchema,
WorkDirRetention: workDirRetention,
TotalLLMConcurrency: totalLLMConcurrency,
ProposalLLMConcurrency: proposalLLMConcurrency,
ValidationModel: validationModel,
ValidationLLMConcurrency: validationLLMConcurrency,
Report: report,
@@ -83,33 +103,39 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
if cfg.Timeout <= 0 {
return nil, fmt.Errorf("audita timeout must be > 0")
}
if len(cfg.Modules) == 0 {
return nil, fmt.Errorf("audita modules must include at least one module")
}
for i, module := range cfg.Modules {
if strings.TrimSpace(module) == "" {
return nil, fmt.Errorf("audita module at index %d is empty", i)
}
}
if strings.TrimSpace(cfg.BaseURL) == "" {
return nil, fmt.Errorf("audita base url is required")
}
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil {
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
if strings.TrimSpace(cfg.BaseURL) != "" {
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil {
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
}
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
}
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
}
if strings.TrimSpace(cfg.Model) == "" {
return nil, fmt.Errorf("audita model is required")
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
}
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
}
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
}
switch strings.TrimSpace(cfg.OutputSchema) {
case "", "bare-segments", "audita-v1":
default:
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
}
switch strings.TrimSpace(cfg.WorkDirRetention) {
case "", "always", "auto", "never":
default:
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
}
modules := make([]string, len(cfg.Modules))
for i, m := range cfg.Modules {
@@ -123,7 +149,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
modules: modules,
baseURL: strings.TrimSpace(cfg.BaseURL),
model: strings.TrimSpace(cfg.Model),
llmConcurrency: cfg.LLMConcurrency,
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
configPath: strings.TrimSpace(cfg.ConfigPath),
outputSchema: strings.TrimSpace(cfg.OutputSchema),
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
totalLLMConcurrency: cfg.TotalLLMConcurrency,
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
validationModel: strings.TrimSpace(cfg.ValidationModel),
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
report: cfg.Report,
@@ -152,7 +183,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
}
reqModules := req.Modules
if len(reqModules) == 0 {
if reqModules == nil {
reqModules = append([]string(nil), r.modules...)
}
args := r.buildArgs(req, reqModules)
@@ -168,14 +199,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
env["AUDITA_LLM_API_KEY"] = credential
credentialPresent = true
}
primaryConcurrencyViaEnv := false
if r.llmConcurrency != nil {
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
primaryConcurrencyViaEnv = true
}
if req.GeneratedConfigPath != "" {
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
}
}
@@ -196,7 +222,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
req.StderrLogPath,
)
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
"%s: %w",
wrappedMessage,
err,
@@ -204,11 +230,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
}
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
}
if r.report {
if err := validateJSONFile(req.ReportPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
}
}
@@ -223,21 +249,25 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
Duration: runRes.Duration,
InvokedBinary: r.binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": reqModules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
"adapter": "audita_subprocess",
"modules": reqModules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
},
}, nil
}
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
return PolishResult{
ProcessedTranscriptPath: req.OutputProcessedPath,
ReportPath: req.ReportPath,
@@ -249,16 +279,20 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
Duration: runRes.Duration,
InvokedBinary: r.binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
"adapter": "audita_subprocess",
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
},
}
}
@@ -269,14 +303,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--modules", strings.Join(modules, ","),
"--base-url", r.baseURL,
"--model", r.model,
"--work-dir", req.WorkDir,
}
if r.baseURL != "" {
args = append(args, "--base-url", r.baseURL)
}
if r.model != "" {
args = append(args, "--model", r.model)
}
if len(modules) > 0 {
args = append(args, "--modules", strings.Join(modules, ","))
}
if r.report {
args = append(args, "--report-json", req.ReportPath)
}
if r.transcriptDescription != "" {
args = append(args, "--transcript-description", r.transcriptDescription)
}
if r.configPath != "" {
args = append(args, "--config", r.configPath)
}
if r.outputSchema != "" {
args = append(args, "--output-schema", r.outputSchema)
}
if r.workDirRetention != "" {
args = append(args, "--work-dir-retention", r.workDirRetention)
}
if r.totalLLMConcurrency != nil {
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
}
if r.proposalLLMConcurrency != nil {
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
}
if r.validationModel != "" {
args = append(args, "--validation-model", r.validationModel)
}
@@ -286,29 +344,31 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
return args
}
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
payload := map[string]any{
"schema": "audita.generated.v1",
"binary": r.binary,
"args": args,
"timeout": r.timeout.String(),
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"validation_model": r.validationModel,
"validation_llm_concurrency": r.validationLLMConcurrency,
"report_enabled": r.report,
"merged_transcript_path": req.MergedTranscriptPath,
"glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath,
"report_path": req.ReportPath,
"work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
}
if r.llmConcurrency != nil {
payload["llm_concurrency"] = *r.llmConcurrency
"schema": "audita.generated.v1",
"binary": r.binary,
"args": args,
"timeout": r.timeout.String(),
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"report_enabled": r.report,
"merged_transcript_path": req.MergedTranscriptPath,
"glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath,
"report_path": req.ReportPath,
"work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
}

View File

@@ -25,7 +25,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
wrapper := writeAuditaHelperWrapper(t)
llmConcurrency := 1
totalLLMConcurrency := 3
proposalLLMConcurrency := 2
validationLLMConcurrency := 2
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
Binary: wrapper,
@@ -34,7 +35,12 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
Modules: []string{"glossary", "homophones", "glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
TranscriptDescription: "Campaign Session 42",
ConfigPath: "/etc/audita/config.yml",
OutputSchema: "audita-v1",
WorkDirRetention: "auto",
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
ValidationModel: "openrouter/google/gemma-4-31b-it",
ValidationLLMConcurrency: &validationLLMConcurrency,
Report: true,
@@ -93,11 +99,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
"process", req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--modules", "glossary,homophones,glossary",
"--work-dir", req.WorkDir,
"--base-url", "https://openrouter.ai/api/v1",
"--model", "openrouter/google/gemma-4-31b-it",
"--work-dir", req.WorkDir,
"--modules", "glossary,homophones,glossary",
"--report-json", req.ReportPath,
"--transcript-description", "Campaign Session 42",
"--config", "/etc/audita/config.yml",
"--output-schema", "audita-v1",
"--work-dir-retention", "auto",
"--total-llm-concurrency", "3",
"--proposal-llm-concurrency", "2",
"--validation-model", "openrouter/google/gemma-4-31b-it",
"--validation-llm-concurrency", "2",
}
@@ -107,8 +119,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
}
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
}
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
@@ -124,16 +136,14 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
@@ -155,16 +165,14 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
@@ -211,6 +219,65 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
}
}
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--modules" {
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--base-url" {
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
}
if rec.Args[i] == "--model" {
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
@@ -220,16 +287,14 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -256,16 +321,14 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -286,16 +349,14 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -316,16 +377,14 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -346,16 +405,14 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: false,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
@@ -376,16 +433,14 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
llmConcurrency := 1
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
Report: true,
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
@@ -398,11 +453,11 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
}
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected binary validation error")
}
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected timeout parse error")
}

View File

@@ -0,0 +1,29 @@
package storage
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// NewObjectStoreFromConfig constructs a remote object store from resolved config.
func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) {
if cfg == nil || cfg.Pipeline == nil {
return nil, fmt.Errorf("pipeline config is required")
}
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
if cfg.Pipeline.Storage.S3 == nil {
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
}
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
return nil, fmt.Errorf("no remote object store backend is configured")
}

View File

@@ -0,0 +1,56 @@
package storage
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
return &fakeS3API{}, nil
}
store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{
Backend: "s3",
S3: &config.StorageS3Config{
Bucket: "my-archive",
},
},
},
})
if err != nil {
t.Fatalf("NewObjectStoreFromConfig() error = %v", err)
}
if _, ok := store.(*S3Backend); !ok {
t.Fatalf("store type = %T, want *S3Backend", store)
}
}
func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{Backend: "s3"},
},
})
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err)
}
}
func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{Backend: "local"},
},
})
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
}
}

View File

@@ -1,6 +1,14 @@
package storage
import "context"
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// NoopBackend is a deterministic no-op archive/storage adapter.
type NoopBackend struct{}
@@ -18,6 +26,21 @@ type FakeBackend struct {
Requests []ArchiveRequest
Err error
Result ArchiveResult
Objects map[string]FakeObject
Uploads []FakeUploadCall
ListErr error
DownloadErr error
UploadErr error
ExistsErr error
}
// FakeUploadCall captures one upload invocation in call order.
type FakeUploadCall struct {
LocalPath string
Key string
Options UploadOptions
}
// Archive records request and returns configured response.
@@ -38,3 +61,148 @@ func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveR
}
return res, nil
}
// FakeObject is a deterministic fake object-store record.
type FakeObject struct {
Key string
Data []byte
Metadata map[string]string
ETag string
LastModified *time.Time
}
// SeedObject inserts or replaces an object in the fake object store.
func (f *FakeBackend) SeedObject(obj FakeObject) {
if f.Objects == nil {
f.Objects = map[string]FakeObject{}
}
key := normalizeObjectKey(obj.Key)
obj.Key = key
obj.Data = append([]byte(nil), obj.Data...)
obj.Metadata = copyMetadata(obj.Metadata)
f.Objects[key] = obj
}
// List returns deterministic prefix-filtered objects.
func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if f.ListErr != nil {
return nil, f.ListErr
}
normalizedPrefix := normalizeObjectKey(prefix)
keys := make([]string, 0, len(f.Objects))
for key := range f.Objects {
if strings.HasPrefix(key, normalizedPrefix) {
keys = append(keys, key)
}
}
sort.Strings(keys)
out := make([]ObjectInfo, 0, len(keys))
for _, key := range keys {
obj := f.Objects[key]
out = append(out, ObjectInfo{
Key: obj.Key,
Size: int64(len(obj.Data)),
ETag: obj.ETag,
LastModified: obj.LastModified,
})
}
return out, nil
}
// Download writes one object to a local path.
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
if err := ctx.Err(); err != nil {
return err
}
if f.DownloadErr != nil {
return f.DownloadErr
}
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
obj, ok := f.Objects[normalizeObjectKey(key)]
if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
}
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
}
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
return fmt.Errorf("download object %q: write local file: %w", key, err)
}
return nil
}
// Upload reads a local file and stores it under key.
func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, err
}
if f.UploadErr != nil {
return ObjectInfo{}, f.UploadErr
}
if strings.TrimSpace(localPath) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
}
if strings.TrimSpace(key) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := os.ReadFile(localPath)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
}
normalizedKey := normalizeObjectKey(key)
f.Uploads = append(f.Uploads, FakeUploadCall{
LocalPath: localPath,
Key: normalizedKey,
Options: UploadOptions{
Metadata: copyMetadata(opts.Metadata),
ContentType: opts.ContentType,
},
})
now := time.Now().UTC()
obj := FakeObject{
Key: normalizedKey,
Data: data,
Metadata: copyMetadata(opts.Metadata),
LastModified: &now,
}
f.SeedObject(obj)
return ObjectInfo{
Key: normalizedKey,
Size: int64(len(data)),
LastModified: &now,
}, nil
}
// Exists checks object presence.
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
if f.ExistsErr != nil {
return false, f.ExistsErr
}
_, ok := f.Objects[normalizeObjectKey(key)]
return ok, nil
}
func copyMetadata(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for k, v := range in {
out[k] = v
}
return out
}

View File

@@ -3,6 +3,9 @@ package storage
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -29,3 +32,84 @@ func TestFakeBackendError(t *testing.T) {
t.Fatal("expected error, got nil")
}
}
func TestFakeBackendListPrefixFiltering(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")})
fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")})
items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/")
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 2 {
t.Fatalf("List() len = %d, want 2", len(items))
}
if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" {
t.Fatalf("List() keys = %#v", items)
}
}
func TestFakeBackendDownload(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
t.Fatalf("Download() error = %v", err)
}
data, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "audio-a" {
t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a")
}
}
func TestFakeBackendUploadAndExists(t *testing.T) {
fake := &FakeBackend{}
local := filepath.Join(t.TempDir(), "upload.txt")
if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
Metadata: map[string]string{"kind": "artifact"},
})
if err != nil {
t.Fatalf("Upload() error = %v", err)
}
if info.Key != "runs/id/artifact.txt" {
t.Fatalf("Upload() key = %q, want normalized key", info.Key)
}
ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if !ok {
t.Fatal("Exists() = false, want true")
}
}
func TestFakeBackendObjectErrors(t *testing.T) {
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
t.Fatalf("List() error = %v, want list fail", err)
}
if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") {
t.Fatalf("Download() error = %v, want download fail", err)
}
local := filepath.Join(t.TempDir(), "x.txt")
_ = os.WriteFile(local, []byte("x"), 0o644)
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
t.Fatalf("Upload() error = %v, want upload fail", err)
}
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
t.Fatalf("Exists() error = %v, want exists fail", err)
}
}

View File

@@ -0,0 +1,8 @@
package storage
import "strings"
func normalizeObjectKey(key string) string {
normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/")
return strings.TrimLeft(normalized, "/")
}

View File

@@ -0,0 +1,21 @@
package storage
import "testing"
func TestNormalizeObjectKey(t *testing.T) {
tests := []struct {
in string
want string
}{
{in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"},
{in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"},
{in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"},
{in: "", want: ""},
}
for _, tt := range tests {
if got := normalizeObjectKey(tt.in); got != tt.want {
t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}

View File

@@ -0,0 +1,32 @@
package storage
import (
"context"
"time"
)
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
//
// Key invariant:
// callers pass full bucket-relative object keys. Backend implementations do not
// infer Narratio session semantics and do not prepend root prefixes.
type ObjectStore interface {
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
Download(ctx context.Context, key, localPath string) error
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
Exists(ctx context.Context, key string) (bool, error)
}
// ObjectInfo describes one object in remote storage.
type ObjectInfo struct {
Key string
Size int64
ETag string
LastModified *time.Time
}
// UploadOptions configures optional object upload metadata.
type UploadOptions struct {
Metadata map[string]string
ContentType string
}

View File

@@ -0,0 +1,275 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type s3API interface {
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
}
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
type S3Backend struct {
bucket string
client s3API
}
type s3ClientOptions struct {
Region string
Endpoint string
ForcePathStyle bool
AccessKeyID string
SecretKey string
}
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
if strings.TrimSpace(opts.Region) != "" {
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
}
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
strings.TrimSpace(opts.AccessKeyID),
strings.TrimSpace(opts.SecretKey),
"",
),
))
}
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
if err != nil {
return nil, fmt.Errorf("load aws config: %w", err)
}
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if strings.TrimSpace(opts.Endpoint) != "" {
endpoint := strings.TrimSpace(opts.Endpoint)
o.BaseEndpoint = &endpoint
}
o.UsePathStyle = opts.ForcePathStyle
}), nil
}
// NewS3BackendFromConfig builds an S3 backend from resolved config.
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
bucket := strings.TrimSpace(cfg.Bucket)
if bucket == "" {
return nil, fmt.Errorf("storage.s3.bucket is required")
}
client, err := newS3Client(ctx, s3ClientOptions{
Region: cfg.Region,
Endpoint: cfg.Endpoint,
ForcePathStyle: cfg.ForcePathStyle,
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
})
if err != nil {
return nil, fmt.Errorf("build s3 client: %w", err)
}
return &S3Backend{
bucket: bucket,
client: client,
}, nil
}
func s3CredentialFromEnv(envVarName string) string {
name := strings.TrimSpace(envVarName)
if name == "" {
return ""
}
value, ok := os.LookupEnv(name)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func orDefaultEnvName(name, fallback string) string {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
return fallback
}
return trimmed
}
// List returns objects under prefix.
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
normalizedPrefix := normalizeObjectKey(prefix)
out := make([]ObjectInfo, 0)
var token *string
for {
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: &b.bucket,
Prefix: &normalizedPrefix,
ContinuationToken: token,
})
if err != nil {
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
}
for _, item := range resp.Contents {
var lastModified *time.Time
if item.LastModified != nil {
t := *item.LastModified
lastModified = &t
}
out = append(out, ObjectInfo{
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
Size: valueOrZeroInt64(item.Size),
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
LastModified: lastModified,
})
}
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
break
}
token = resp.NextContinuationToken
}
return out, nil
}
// Download retrieves one object to localPath, creating parent directories as needed.
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
normalizedKey := normalizeObjectKey(key)
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
if err != nil {
return fmt.Errorf("download object %q: %w", normalizedKey, err)
}
defer resp.Body.Close()
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
}
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
}
defer dst.Close()
if _, err := io.Copy(dst, resp.Body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
}
if err := dst.Sync(); err != nil {
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
}
return nil
}
// Upload sends a local file to key.
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
normalizedKey := normalizeObjectKey(key)
if strings.TrimSpace(localPath) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
}
if normalizedKey == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
file, err := os.Open(localPath)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
}
input := &s3.PutObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
Body: file,
Metadata: copyMetadata(opts.Metadata),
}
if strings.TrimSpace(opts.ContentType) != "" {
ct := strings.TrimSpace(opts.ContentType)
input.ContentType = &ct
}
resp, err := b.client.PutObject(ctx, input)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
}
return ObjectInfo{
Key: normalizedKey,
Size: stat.Size(),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
}, nil
}
// Exists checks whether one object key exists.
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
normalizedKey := normalizeObjectKey(key)
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
if err == nil {
return true, nil
}
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NotFound", "NoSuchKey", "404":
return false, nil
}
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
}
func valueOrEmpty(v *string) string {
if v == nil {
return ""
}
return *v
}
func valueOrZeroInt64(v *int64) int64 {
if v == nil {
return 0
}
return *v
}
func valueOrFalseBool(v *bool) bool {
if v == nil {
return false
}
return *v
}

View File

@@ -0,0 +1,253 @@
package storage
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type fakeS3API struct {
listOut *s3.ListObjectsV2Output
listErr error
getBody io.ReadCloser
getErr error
putOut *s3.PutObjectOutput
putErr error
headErr error
lastList *s3.ListObjectsV2Input
lastGet *s3.GetObjectInput
lastPut *s3.PutObjectInput
lastHead *s3.HeadObjectInput
}
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
f.lastList = params
if f.listErr != nil {
return nil, f.listErr
}
if f.listOut == nil {
return &s3.ListObjectsV2Output{}, nil
}
return f.listOut, nil
}
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
f.lastGet = params
if f.getErr != nil {
return nil, f.getErr
}
body := f.getBody
if body == nil {
body = io.NopCloser(strings.NewReader(""))
}
return &s3.GetObjectOutput{Body: body}, nil
}
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
f.lastPut = params
if f.putErr != nil {
return nil, f.putErr
}
if f.putOut == nil {
return &s3.PutObjectOutput{}, nil
}
return f.putOut, nil
}
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
f.lastHead = params
if f.headErr != nil {
return nil, f.headErr
}
return &s3.HeadObjectOutput{}, nil
}
func TestS3BackendListAndKeyNormalization(t *testing.T) {
lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC)
client := &fakeS3API{
listOut: &s3.ListObjectsV2Output{
Contents: []types.Object{
{Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified},
},
},
}
backend := &S3Backend{bucket: "bucket-1", client: client}
items, err := backend.List(context.Background(), `dnd\campaigns\`)
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 1 {
t.Fatalf("List() len = %d, want 1", len(items))
}
if items[0].Key != "dnd/campaigns/forsaken/a.flac" {
t.Fatalf("List() key = %q, want normalized slash key", items[0].Key)
}
if items[0].ETag != "abc" {
t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc")
}
if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" {
t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList)
}
}
func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))}
backend := &S3Backend{bucket: "bucket-1", client: client}
dst := filepath.Join(t.TempDir(), "nested", "clip.flac")
if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil {
t.Fatalf("Download() error = %v", err)
}
data, err := os.ReadFile(dst)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "audio" {
t.Fatalf("downloaded content = %q, want %q", string(data), "audio")
}
if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" {
t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet)
}
}
func TestS3BackendUploadAndExists(t *testing.T) {
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
backend := &S3Backend{bucket: "bucket-1", client: client}
local := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
Metadata: map[string]string{"kind": "artifact"},
})
if err != nil {
t.Fatalf("Upload() error = %v", err)
}
if info.Key != "runs/id/artifact.txt" {
t.Fatalf("Upload key = %q, want normalized key", info.Key)
}
if info.ETag != "etag123" {
t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123")
}
if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" {
t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut)
}
ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if !ok {
t.Fatal("Exists() = false, want true")
}
}
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
if err == nil || !strings.Contains(err.Error(), "no such file") {
t.Fatalf("Upload() error = %v, want missing local file error", err)
}
}
func TestS3BackendExistsNotFound(t *testing.T) {
backend := &S3Backend{
bucket: "bucket-1",
client: &fakeS3API{
headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"},
},
}
ok, err := backend.Exists(context.Background(), "missing-key")
if err != nil {
t.Fatalf("Exists() error = %v", err)
}
if ok {
t.Fatal("Exists() = true, want false")
}
}
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
var got s3ClientOptions
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
got = opts
return &fakeS3API{}, nil
}
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
Endpoint: "http://localhost:9000",
ForcePathStyle: true,
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
}
if backend.bucket != "my-archive" {
t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive")
}
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
}
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
}
}
func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{})
if err == nil || !strings.Contains(err.Error(), "bucket is required") {
t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err)
}
}
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
var got s3ClientOptions
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
got = opts
return &fakeS3API{}, nil
}
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
SecretKeyEnv: "MISSING_SECRET_KEY",
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
}
if got.AccessKeyID != "" || got.SecretKey != "" {
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
}
}
func strPtr(v string) *string { return &v }
func int64Ptr(v int64) *int64 { return &v }
var _ s3API = (*fakeS3API)(nil)

View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -63,12 +64,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
args []string
want string
}{
{name: "run missing flags", args: []string{"run"}, want: "run: --config and --session are required"},
{name: "plan missing flags", args: []string{"plan"}, want: "plan: --config and --session are required"},
{name: "run missing flags", args: []string{"run"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "plan missing flags", args: []string{"plan"}, want: "plan: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
{name: "resume missing flags", args: []string{"resume"}, want: "resume: --config and --session are required"},
{name: "resume missing flags", args: []string{"resume"}, want: "resume: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: --config and --session are required"},
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "run missing config uses defaults", args: []string{"run", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
}
for _, tc := range cases {
@@ -168,6 +170,159 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
}
}
func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
sessionID := "2026-05-03"
secretsDir := filepath.Join(configDir, "secrets")
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
}
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err)
}
seriatimBinary := writeSeriatimAppTestWrapper(t)
auditaBinary := writeAuditaAppTestWrapper(t)
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
pipelinePath := filepath.Join(configDir, "pipeline.yml")
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
root: ` + workspaceRoot + `
storage:
backend: local
secrets:
env_dir: ./secrets
whisperx:
transcribe_url: https://example.com/transcribe
seriatim:
binary: ` + seriatimBinary + `
audita:
binary: ` + auditaBinary + `
llm_api_key_env: OPENROUTER_API_KEY
analyzer:
timeout: 20m
notification:
timeout: 10s
`
sessionYAML := `session_id: ` + sessionID + `
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
originalWD, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(configDir); err != nil {
t.Fatalf("Chdir(%q): %v", configDir, err)
}
t.Cleanup(func() {
_ = os.Chdir(originalWD)
})
workRoot := filepath.Join(workspaceRoot, "work", sessionID)
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
t.Fatalf("stdout = %q, want polish execution", stdout.String())
}
}
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml")
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
root: ` + workspaceRoot + `
storage:
backend: local
secrets:
env_dir: ./missing-secrets
whisperx:
transcribe_url: https://example.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
analyzer:
timeout: 20m
notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "read secrets env_dir") {
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
}
}
func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
workspaceRoot := t.TempDir()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"source":"default-config-test","segments":[{"speaker":"alice"}]}`))
}))
defer srv.Close()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
defer func() {
config.DefaultPipelineConfigSearchPaths = originalDefaults
}()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "--session", sessionPath}, &stdout, &stderr)
if code != 0 {
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=9 skipped=0; manifest=") {
t.Fatalf("stdout = %q, want successful run output", stdout.String())
}
}
func TestExecuteInvalidCommand(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -224,6 +379,11 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
root: ` + workspaceRoot + `
storage:
backend: s3
s3:
bucket: test-bucket
archive:
enabled: true
upload_run: false
whisperx:
transcribe_url: ` + url + `
timeout: 2s
@@ -247,6 +407,7 @@ notification:
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml

View File

@@ -0,0 +1,49 @@
package app
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func resolvePipelineConfigPath(flagValue string) (string, error) {
return resolvePipelineConfigPathWithCandidates(flagValue, config.DefaultPipelineConfigSearchPaths)
}
func resolvePipelineConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
if explicit := strings.TrimSpace(flagValue); explicit != "" {
return explicit, nil
}
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
if path == "" {
continue
}
ordered = append(ordered, path)
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default pipeline config %q: %w", path, err)
}
if len(ordered) == 0 {
return "", fmt.Errorf("no pipeline config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no pipeline config path provided and no default pipeline config found; searched: %s",
strings.Join(ordered, ", "),
)
}

View File

@@ -0,0 +1,65 @@
package app
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolvePipelineConfigPathWithCandidatesExplicitWins(t *testing.T) {
got, err := resolvePipelineConfigPathWithCandidates(" ./custom/pipeline.yml ", []string{"/a", "/b"})
if err != nil {
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
}
if got != "./custom/pipeline.yml" {
t.Fatalf("resolved path = %q, want explicit path", got)
}
}
func TestResolvePipelineConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(second) {
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
}
}
func TestResolvePipelineConfigPathWithCandidatesPrecedence(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(first, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
t.Fatalf("write first default: %v", err)
}
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(first) {
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
}
}
func TestResolvePipelineConfigPathWithCandidatesMissing(t *testing.T) {
_, err := resolvePipelineConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "no default pipeline config found") {
t.Fatalf("error = %q, want missing-defaults context", err.Error())
}
}

View File

@@ -5,9 +5,12 @@ import (
"flag"
"fmt"
"io"
"log/slog"
"os"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/logging"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -18,9 +21,11 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -29,17 +34,27 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("plan: unexpected positional arguments")
}
if pipelinePath == "" || sessionPath == "" {
return fmt.Errorf("plan: --config and --session are required")
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("plan: %w", err)
}
cfg, err := config.Load(pipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("plan: %w", err)
}
if err := config.Validate(cfg); err != nil {
return fmt.Errorf("plan: %w", err)
}
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
return fmt.Errorf("plan: %w", err)
}
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
paths, err := store.EnsureLayout(cfg.Session.SessionID)

View File

@@ -89,6 +89,54 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
}
}
func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml")
sessionPath := filepath.Join(configDir, "session.yml")
pipelineYAML := `workspace:
root: ` + workspaceRoot + `
storage:
backend: local
secrets:
env_dir: ./missing-secrets
whisperx:
transcribe_url: https://example.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
analyzer:
timeout: 20m
notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatalf("write pipeline.yml: %v", err)
}
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "read secrets env_dir") {
t.Fatalf("error = %q, want secrets read error context", err.Error())
}
}
func assertDir(t *testing.T, path string) {
t.Helper()
info, err := os.Stat(path)

View File

@@ -0,0 +1,208 @@
package app
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
return nil
}
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterArchive
workRequested := env.Config.Pipeline.Workspace.CleanupAfterArchive
if !spoolRequested && !workRequested {
return nil
}
sr := archiveStageRecordForCleanup(m, executed)
if sr == nil {
return nil
}
if sr.Metadata == nil {
sr.Metadata = map[string]any{}
}
sr.Metadata["spool_cleanup_requested"] = spoolRequested
sr.Metadata["workdir_cleanup_requested"] = workRequested
eligible, reason := archiveCleanupEligible(env.Config, sr)
if !eligible {
sr.Metadata["cleanup_skipped"] = true
sr.Metadata["cleanup_skipped_reason"] = reason
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
}
return nil
}
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolDir == "" {
spoolDir = artifacts.SessionSpoolAudioDir(
env.Config.Pipeline.Spool.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
)
}
workDir := strings.TrimSpace(m.LocalWorkDir)
if workDir == "" {
workDir = artifacts.SessionRunWorkDir(
env.Config.Pipeline.Workspace.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
)
}
if spoolRequested {
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_archive"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_archive"
sr.Metadata["cleanup_failed_path"] = spoolDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
}
if !workRequested {
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
}
return nil
}
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_archive"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_archive"
sr.Metadata["cleanup_failed_path"] = workDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
return nil
}
func archiveStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
if m == nil {
return nil
}
archiveRan := false
for _, name := range executed {
if name == "archive" {
archiveRan = true
break
}
}
if !archiveRan {
return nil
}
sr := m.Stages["archive"]
if sr == nil || sr.Status != manifest.StatusSucceeded {
return nil
}
return sr
}
func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
return false, "archive configuration is missing"
}
enabled := true
if cfg.Pipeline.Archive.Enabled != nil {
enabled = *cfg.Pipeline.Archive.Enabled
}
if !enabled {
return false, "archive.enabled is false"
}
uploadRun := true
if cfg.Pipeline.Archive.UploadRun != nil {
uploadRun = *cfg.Pipeline.Archive.UploadRun
}
if !uploadRun {
return false, "archive.upload_run is false"
}
if sr == nil || sr.Metadata == nil {
return false, "archive metadata is missing"
}
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
return false, "archive stage was skipped"
}
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
return false, "archive did not upload run record"
}
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
return false, "archive did not write current pointer"
}
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
return false, "archive current run pointer key is missing"
}
return true, ""
}
func removeRunScopedDir(root, target, policy string) error {
cleanRoot := strings.TrimSpace(root)
cleanTarget := strings.TrimSpace(target)
if cleanRoot == "" {
return fmt.Errorf("cleanup policy %s: root path is required", policy)
}
if cleanTarget == "" {
return fmt.Errorf("cleanup policy %s: target path is required", policy)
}
rootAbs, err := filepath.Abs(cleanRoot)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
}
targetAbs, err := filepath.Abs(cleanTarget)
if err != nil {
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
}
rel, err := filepath.Rel(rootAbs, targetAbs)
if err != nil {
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
}
if rel == "." {
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
}
info, err := os.Lstat(targetAbs)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
}
if !info.IsDir() {
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
}
if err := os.RemoveAll(targetAbs); err != nil {
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
}
return nil
}
func asString(v any) string {
s, _ := v.(string)
return s
}

View File

@@ -0,0 +1,396 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type archiveSuccessStage struct {
metadata map[string]any
}
func (archiveSuccessStage) Name() string { return "archive" }
func (archiveSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
func (s archiveSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
md := map[string]any{
"stage": "archive",
"uploaded": true,
"current_pointer_written": true,
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
}
for k, v := range s.metadata {
md[k] = v
}
return &stage.StageResult{Metadata: md}, nil
}
type notifyFailStage struct{}
func (notifyFailStage) Name() string { return "notify" }
func (notifyFailStage) Declares() stage.IODecl { return stage.IODecl{} }
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
return nil, errors.New("notify failed")
}
func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
cfg.Pipeline.Workspace.CleanupAfterArchive = false
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, seed.localSourceAudio)
}
func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = false
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertMissing(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, seed.localSourceAudio)
}
func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, cfg.Pipeline.Workspace.Root)
assertExists(t, seed.otherRunDir)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.spoolAudioDir)
}
func TestPostArchiveCleanupBothPolicies(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertMissing(t, seed.spoolAudioDir)
assertMissing(t, seed.runWorkDir)
assertExists(t, seed.otherRunDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "archive", err: errors.New("archive failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "stage \"archive\" failed") {
t.Fatalf("executeStages() error = %v, want archive failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
cfg.Pipeline.Archive.UploadRun = boolPtr(false)
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
cfg, seed := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
_, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
t.Fatalf("executeStages() error = %v, want notify failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
cfg, _ := cleanupFixtureConfig(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = false
manifestPath := manifestPathFor(cfg)
store := &manifest.LocalStore{}
m, err := store.Load(context.Background(), manifestPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
m.LocalSpoolDir = filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool")
if err := store.Save(context.Background(), manifestPath, m); err != nil {
t.Fatalf("Save() error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
}
}
func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
cfg, seed, runID := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
cfg.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
}
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
t.Fatalf("executeStages() error = %v, want promotion-missing failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
assertExists(t, artifacts.SessionRunWorkDir(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
}
func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
cfg, seed, _ := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
failKey := seed.sessionPrefix + "current/manifest.json"
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
})
if err == nil || !strings.Contains(err.Error(), "current manifest") {
t.Fatalf("executeStages() error = %v, want current-manifest failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
cfg, seed, _ := archiveStageCleanupFixture(t)
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
cfg.Pipeline.Workspace.CleanupAfterArchive = true
failKey := seed.sessionPrefix + "current/run_id.txt"
archiveStageImpl, err := stage.Select("archive")
if err != nil {
t.Fatalf("Select(archive) error = %v", err)
}
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
})
if err == nil || !strings.Contains(err.Error(), "current run pointer") {
t.Fatalf("executeStages() error = %v, want current-run-pointer failure", err)
}
assertExists(t, seed.spoolAudioDir)
assertExists(t, seed.runWorkDir)
}
type cleanupSeed struct {
runWorkDir string
otherRunDir string
spoolAudioDir string
localSourceAudio string
sessionPrefix string
}
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
t.Helper()
cfg := testConfig(t)
cfg.Pipeline.Archive = &config.ArchiveConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
runID := "20260516T010203Z-1a2b3c4d"
runWorkDir := artifacts.SessionRunWorkDir(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
otherRunDir := artifacts.SessionRunWorkDir(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
mustWriteFile(t, localSourceAudio, "source\n")
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.Campaign = cfg.Session.Campaign
seed.RunID = runID
seed.LocalWorkDir = runWorkDir
seed.LocalSpoolDir = spoolAudioDir
seed.S3Bucket = "my-dnd-archive"
seed.S3SessionPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/"
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
store := &manifest.LocalStore{}
if err := os.MkdirAll(filepath.Dir(manifestPathFor(cfg)), 0o755); err != nil {
t.Fatalf("MkdirAll() error = %v", err)
}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatalf("seed manifest save error = %v", err)
}
return cfg, cleanupSeed{
runWorkDir: runWorkDir,
otherRunDir: otherRunDir,
spoolAudioDir: spoolAudioDir,
localSourceAudio: localSourceAudio,
sessionPrefix: seed.S3SessionPrefix,
}
}
func archiveStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, string) {
t.Helper()
cfg, seed := cleanupFixtureConfig(t)
runID := "20260516T010203Z-1a2b3c4d"
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
cfg.Pipeline.Archive = &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
PromoteArtifacts: []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
},
}
writeArchiveFixtureRunFiles(t, seed.runWorkDir)
store := &manifest.LocalStore{}
seedManifest, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatalf("Load() error = %v", err)
}
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)
if err := store.Save(context.Background(), manifestPathFor(cfg), seedManifest); err != nil {
t.Fatalf("Save() error = %v", err)
}
seed.sessionPrefix = seedManifest.S3SessionPrefix
return cfg, seed, runID
}
func writeArchiveFixtureRunFiles(t *testing.T, runWorkDir string) {
t.Helper()
mustWriteFile(t, filepath.Join(runWorkDir, "inputs", "session.yml"), "session_id: 2026-05-03\n")
mustWriteFile(t, filepath.Join(runWorkDir, "transcripts", "raw", "speaker.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "transcripts", "trimmed.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "artifacts", "session_recap.md"), "# recap\n")
mustWriteFile(t, filepath.Join(runWorkDir, "reports", "audita.report.json"), "{}\n")
mustWriteFile(t, filepath.Join(runWorkDir, "config", "audita.generated.yml"), "key: value\n")
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
}
type failKeyStore struct {
delegate *storage.FakeBackend
failKey string
}
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
return s.delegate.List(ctx, prefix)
}
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
return s.delegate.Download(ctx, key, localPath)
}
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
return storage.ObjectInfo{}, errors.New("forced upload failure")
}
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
return s.delegate.Exists(ctx, key)
}
func assertExists(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected path to exist %q: %v", path, err)
}
}
func assertMissing(t *testing.T, path string) {
t.Helper()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
}
}

View File

@@ -17,9 +17,11 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution")
if err := fs.Parse(args); err != nil {
@@ -28,11 +30,18 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("resume: unexpected positional arguments")
}
if pipelinePath == "" || sessionPath == "" {
return fmt.Errorf("resume: --config and --session are required")
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("resume: %w", err)
}
cfg, err := config.Load(pipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("resume: %w", err)
}

View File

@@ -16,9 +16,11 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -27,11 +29,18 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 0 {
return fmt.Errorf("run: unexpected positional arguments")
}
if pipelinePath == "" || sessionPath == "" {
return fmt.Errorf("run: --config and --session are required")
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run: %w", err)
}
cfg, err := config.Load(pipelinePath, sessionPath)
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("run: %w", err)
}

View File

@@ -16,9 +16,11 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
var pipelinePath string
var sessionPath string
var sessionID string
var force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
if err := fs.Parse(args); err != nil {
@@ -27,17 +29,24 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
if fs.NArg() != 1 {
return fmt.Errorf("run-stage: expected exactly one stage name")
}
if pipelinePath == "" || sessionPath == "" {
return fmt.Errorf("run-stage: --config and --session are required")
}
stageName := fs.Arg(0)
stages, err := BuildSingleStagePlan(stageName)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
cfg, err := config.Load(pipelinePath, sessionPath)
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
SessionID: sessionID,
})
if err != nil {
return fmt.Errorf("run-stage: %w", err)
}

View File

@@ -51,6 +51,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if env.Logger == nil {
env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
}
if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
return nil, fmt.Errorf("load secrets from files: %w", err)
}
if env.WhisperX == nil {
client, err := buildDefaultWhisperXClient(env.Config)
if err != nil {
@@ -78,6 +81,13 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if env.Storage == nil {
env.Storage = &storage.NoopBackend{}
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
if err != nil {
return nil, fmt.Errorf("initialize object store backend: %w", err)
}
env.ObjectStore = objectStore
}
if env.Notifier == nil {
env.Notifier = &notify.NoopSender{}
}
@@ -101,6 +111,15 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
if err != nil {
return nil, err
}
identityChanged, err := ensureManifestIdentity(cfg, m)
if err != nil {
return nil, fmt.Errorf("initialize manifest identity: %w", err)
}
if identityChanged {
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return nil, fmt.Errorf("save manifest identity %q: %w", manifestPath, err)
}
}
stageEnv := env
@@ -149,6 +168,10 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
env.Logger.Info("stage succeeded", "stage", s.Name())
}
if err := runPostArchiveCleanup(ctx, env, manifestPath, m, executed); err != nil {
return nil, fmt.Errorf("post-archive cleanup: %w", err)
}
return &RunSummary{
SessionID: cfg.Session.SessionID,
ManifestPath: manifestPath,
@@ -227,7 +250,7 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
}
a := cfg.Pipeline.Audita
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" || len(a.Modules) == 0 || strings.TrimSpace(a.BaseURL) == "" || strings.TrimSpace(a.Model) == "" {
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" {
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
return &audita.NoopRunner{}, nil
}
@@ -244,7 +267,12 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
append([]string(nil), a.Modules...),
a.BaseURL,
a.Model,
a.LLMConcurrency,
a.TranscriptDescription,
a.ConfigPath,
a.OutputSchema,
a.WorkDirRetention,
a.TotalLLMConcurrency,
a.ProposalLLMConcurrency,
a.ValidationModel,
a.ValidationLLMConcurrency,
report,
@@ -330,6 +358,89 @@ func applyStageResultToManifest(m *manifest.Manifest, stageName string, result *
}
}
func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest) (bool, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil || m == nil {
return false, nil
}
changed := false
campaign := strings.TrimSpace(cfg.Session.Campaign)
sessionID := strings.TrimSpace(cfg.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
if m.Campaign == "" && campaign != "" {
m.Campaign = campaign
changed = true
}
if m.RunID == "" {
runID, err := artifacts.NewRunID()
if err != nil {
return false, err
}
m.RunID = runID
changed = true
}
if m.LocalWorkDir == "" && campaign != "" && sessionID != "" && m.RunID != "" {
m.LocalWorkDir = artifacts.SessionRunWorkDir(cfg.Pipeline.Workspace.Root, campaign, sessionID, m.RunID)
changed = true
}
if m.LocalSpoolDir == "" && campaign != "" && sessionID != "" && m.RunID != "" && strings.TrimSpace(cfg.Pipeline.Spool.Root) != "" {
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, campaign, sessionID, m.RunID)
changed = true
}
if cfg.Pipeline.Storage.S3 != nil {
if m.S3Bucket == "" && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
m.S3Bucket = strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket)
changed = true
}
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
if m.S3SessionPrefix == "" && sessionPrefix != "" {
m.S3SessionPrefix = sessionPrefix
changed = true
}
runPrefix := artifacts.S3RunPrefix(sessionPrefix, m.RunID)
if m.S3RunPrefix == "" && runPrefix != "" {
m.S3RunPrefix = runPrefix
changed = true
}
}
return changed, nil
}
func manifestPathFor(cfg *config.Config) string {
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
}
func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return false
}
stageRequested := func(name string) bool {
for _, s := range stages {
if s != nil && s.Name() == name {
return true
}
}
return false
}
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
return true
}
if !stageRequested("archive") {
return false
}
if cfg.Pipeline.Archive == nil {
return false
}
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
return false
}
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
return false
}
return true
}

View File

@@ -144,6 +144,15 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
}
continue
}
if name == "archive" {
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
}
if sr.Metadata["skipped"] != true {
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
}
continue
}
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
t.Fatalf("stage %q missing placeholder metadata", name)
}
@@ -315,7 +324,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
{name: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("archive fail")}}},
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
{name: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("notify fail")}}},
}
@@ -400,6 +409,41 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
},
}
}
if tc.name == "archive" {
cfg.Pipeline.Archive = &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
}
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
}
runID := "20260516T010203Z-0a1b2c3d"
runWorkDir := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, runID)
if err := os.MkdirAll(filepath.Join(runWorkDir, "inputs"), 0o755); err != nil {
t.Fatalf("mkdir archive inputs dir: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "inputs", "session.yml"), []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write archive fixture session.yml: %v", err)
}
if err := os.WriteFile(filepath.Join(runWorkDir, "manifest.json"), []byte("{}\n"), 0o644); err != nil {
t.Fatalf("write archive fixture manifest.json: %v", err)
}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.Campaign = cfg.Session.Campaign
seed.RunID = runID
seed.LocalWorkDir = runWorkDir
seed.S3Bucket = "my-dnd-archive"
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatalf("seed archive manifest: %v", err)
}
}
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
if runErr == nil {
@@ -430,7 +474,7 @@ func testConfig(t *testing.T) *config.Config {
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\n")
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
@@ -442,6 +486,7 @@ func testConfig(t *testing.T) *config.Config {
SessionPath: sessionPath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",
@@ -452,6 +497,55 @@ func testConfig(t *testing.T) *config.Config {
}
}
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
sessionPath := filepath.Join(dir, "session.yml")
pipelineYAML := `workspace:
root: ` + t.TempDir() + `
whisperx:
transcribe_url: https://example.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 10s
`
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
mustWriteFile(t, pipelinePath, pipelineYAML)
mustWriteFile(t, sessionPath, sessionYAML)
cfg, err := config.Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := config.Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
serRunner, err := buildDefaultSeriatimRunner(cfg)
if err != nil {
t.Fatalf("buildDefaultSeriatimRunner() error = %v", err)
}
if _, ok := serRunner.(*seriatim.SubprocessRunner); !ok {
t.Fatalf("seriatim runner type = %T, want *seriatim.SubprocessRunner", serRunner)
}
audRunner, err := buildDefaultAuditaRunner(cfg)
if err != nil {
t.Fatalf("buildDefaultAuditaRunner() error = %v", err)
}
if _, ok := audRunner.(*audita.SubprocessRunner); !ok {
t.Fatalf("audita runner type = %T, want *audita.SubprocessRunner", audRunner)
}
}
func mustWriteFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
@@ -461,3 +555,8 @@ func mustWriteFile(t *testing.T, path, contents string) {
t.Fatalf("WriteFile(%q): %v", path, err)
}
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -0,0 +1,88 @@
package app
import (
"fmt"
"log/slog"
"os"
"path/filepath"
"regexp"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type secretsLoadStats struct {
Dir string
Loaded int
PreservedExisting int
Skipped int
}
func loadSecretsFromConfig(cfg *config.Config, logger *slog.Logger) (*secretsLoadStats, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Secrets == nil {
return nil, nil
}
rawDir := strings.TrimSpace(cfg.Pipeline.Secrets.EnvDir)
if rawDir == "" {
return nil, nil
}
resolvedDir := rawDir
if !filepath.IsAbs(resolvedDir) {
cwd, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("resolve secrets env_dir %q from current working directory: %w", rawDir, err)
}
resolvedDir = filepath.Join(cwd, resolvedDir)
}
resolvedDir = filepath.Clean(resolvedDir)
entries, err := os.ReadDir(resolvedDir)
if err != nil {
return nil, fmt.Errorf("read secrets env_dir %q: %w", resolvedDir, err)
}
stats := &secretsLoadStats{Dir: resolvedDir}
for _, entry := range entries {
name := entry.Name()
if !envVarNamePattern.MatchString(name) {
stats.Skipped++
continue
}
if entry.IsDir() {
stats.Skipped++
continue
}
secretPath := filepath.Join(resolvedDir, name)
bytes, err := os.ReadFile(secretPath)
if err != nil {
return nil, fmt.Errorf("read secret file %q: %w", secretPath, err)
}
value := strings.TrimRight(string(bytes), "\r\n")
if _, exists := os.LookupEnv(name); exists {
stats.PreservedExisting++
continue
}
if err := os.Setenv(name, value); err != nil {
return nil, fmt.Errorf("set environment variable %q from %q: %w", name, secretPath, err)
}
stats.Loaded++
}
if logger != nil {
logger.Info(
"loaded secret environment variables from filesystem",
"secrets_env_dir", stats.Dir,
"loaded", stats.Loaded,
"preserved_existing", stats.PreservedExisting,
"skipped", stats.Skipped,
)
}
return stats, nil
}

View File

@@ -0,0 +1,155 @@
package app
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestLoadSecretsFromConfigLoadsValidFiles(t *testing.T) {
dir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_A"), "value-1\n")
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_B"), "value-2\r\n")
mustWriteSecretFile(t, filepath.Join(dir, "not-valid-name.txt"), "ignored")
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Secrets: &config.SecretsConfig{EnvDir: dir},
},
}
stats, err := loadSecretsFromConfig(cfg, nil)
if err != nil {
t.Fatalf("loadSecretsFromConfig() error = %v", err)
}
if stats == nil {
t.Fatal("stats = nil, want non-nil")
}
if stats.Loaded != 2 {
t.Fatalf("Loaded = %d, want 2", stats.Loaded)
}
if stats.PreservedExisting != 0 {
t.Fatalf("PreservedExisting = %d, want 0", stats.PreservedExisting)
}
if stats.Skipped == 0 {
t.Fatalf("Skipped = %d, want > 0 for invalid filename", stats.Skipped)
}
if got := os.Getenv("NARRATIO_TEST_SECRET_A"); got != "value-1" {
t.Fatalf("NARRATIO_TEST_SECRET_A = %q, want value-1", got)
}
if got := os.Getenv("NARRATIO_TEST_SECRET_B"); got != "value-2" {
t.Fatalf("NARRATIO_TEST_SECRET_B = %q, want value-2", got)
}
}
func TestLoadSecretsFromConfigPreservesExistingEnv(t *testing.T) {
t.Setenv("OBJECT_STORAGE_KEY", "existing")
dir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(dir, "OBJECT_STORAGE_KEY"), "from-file\n")
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Secrets: &config.SecretsConfig{EnvDir: dir},
},
}
stats, err := loadSecretsFromConfig(cfg, nil)
if err != nil {
t.Fatalf("loadSecretsFromConfig() error = %v", err)
}
if stats.PreservedExisting != 1 {
t.Fatalf("PreservedExisting = %d, want 1", stats.PreservedExisting)
}
if got := os.Getenv("OBJECT_STORAGE_KEY"); got != "existing" {
t.Fatalf("OBJECT_STORAGE_KEY = %q, want existing", got)
}
}
func TestLoadSecretsFromConfigRelativeDirUsesCWD(t *testing.T) {
cwd := t.TempDir()
secretsDir := filepath.Join(cwd, "secrets")
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
}
mustWriteSecretFile(t, filepath.Join(secretsDir, "OBJECT_STORAGE_KEY_ID"), "id-123\n")
originalWD, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(cwd); err != nil {
t.Fatalf("Chdir(%q): %v", cwd, err)
}
t.Cleanup(func() {
_ = os.Chdir(originalWD)
})
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Secrets: &config.SecretsConfig{EnvDir: "./secrets"},
},
}
if _, err := loadSecretsFromConfig(cfg, nil); err != nil {
t.Fatalf("loadSecretsFromConfig() error = %v", err)
}
if got := os.Getenv("OBJECT_STORAGE_KEY_ID"); got != "id-123" {
t.Fatalf("OBJECT_STORAGE_KEY_ID = %q, want id-123", got)
}
}
func TestLoadSecretsFromConfigMissingDirFails(t *testing.T) {
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Secrets: &config.SecretsConfig{EnvDir: filepath.Join(t.TempDir(), "missing")},
},
}
_, err := loadSecretsFromConfig(cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "read secrets env_dir") {
t.Fatalf("error = %q, want read-dir context", err.Error())
}
}
func TestLoadSecretsFromConfigUnreadableValidEntryFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("symlink behavior differs on windows")
}
dir := t.TempDir()
broken := filepath.Join(dir, "OPENROUTER_API_KEY")
if err := os.Symlink(filepath.Join(dir, "does-not-exist"), broken); err != nil {
t.Fatalf("Symlink(%q): %v", broken, err)
}
cfg := &config.Config{
Pipeline: &config.PipelineConfig{
Secrets: &config.SecretsConfig{EnvDir: dir},
},
}
_, err := loadSecretsFromConfig(cfg, nil)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "read secret file") {
t.Fatalf("error = %q, want read secret file context", err.Error())
}
}
func mustWriteSecretFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", path, err)
}
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatalf("WriteFile(%q): %v", path, err)
}
}

View File

@@ -0,0 +1,86 @@
package app
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func TestPlanUsesDiscoveredSessionTemplateWithSessionID(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
sessionTemplate := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionTemplate), 0o644); err != nil {
t.Fatalf("write session template: %v", err)
}
cwd := filepath.Dir(sessionPath)
originalWD, err := os.Getwd()
if err != nil {
t.Fatalf("Getwd(): %v", err)
}
if err := os.Chdir(cwd); err != nil {
t.Fatalf("Chdir(%q): %v", cwd, err)
}
t.Cleanup(func() { _ = os.Chdir(originalWD) })
var out bytes.Buffer
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session-id", "2026-04-04"}, &out); err != nil {
t.Fatalf("Plan() error = %v", err)
}
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
t.Fatalf("output = %q, want plan output", out.String())
}
}
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
if err != nil {
t.Fatalf("RunStage() error = %v", err)
}
if !strings.Contains(out.String(), "stage=prepare") {
t.Fatalf("output = %q, want stage output", out.String())
}
}
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "searched") {
t.Fatalf("error = %q, want searched paths", err.Error())
}
if !strings.Contains(err.Error(), "pass --session") {
t.Fatalf("error = %q, want explicit-session guidance", err.Error())
}
}

View File

@@ -0,0 +1,49 @@
package app
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func resolveSessionConfigPath(flagValue string) (string, error) {
return resolveSessionConfigPathWithCandidates(flagValue, config.DefaultSessionConfigSearchPaths)
}
func resolveSessionConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
if explicit := strings.TrimSpace(flagValue); explicit != "" {
return explicit, nil
}
ordered := make([]string, 0, len(candidates))
for _, raw := range candidates {
path := strings.TrimSpace(raw)
if path == "" {
continue
}
ordered = append(ordered, path)
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
continue
}
return filepath.Clean(path), nil
}
if errors.Is(err, os.ErrNotExist) {
continue
}
return "", fmt.Errorf("check default session config %q: %w", path, err)
}
if len(ordered) == 0 {
return "", fmt.Errorf("no session config path provided and no default locations configured")
}
return "", fmt.Errorf(
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
strings.Join(ordered, ", "),
)
}

View File

@@ -0,0 +1,68 @@
package app
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveSessionConfigPathWithCandidatesExplicitWins(t *testing.T) {
got, err := resolveSessionConfigPathWithCandidates(" ./custom/session.yml ", []string{"./session.yml", "/a", "/b"})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != "./custom/session.yml" {
t.Fatalf("resolved path = %q, want explicit path", got)
}
}
func TestResolveSessionConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(second) {
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
}
}
func TestResolveSessionConfigPathWithCandidatesPrecedence(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "first.yml")
second := filepath.Join(dir, "second.yml")
if err := os.WriteFile(first, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write first default: %v", err)
}
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
t.Fatalf("write second default: %v", err)
}
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
if err != nil {
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
}
if got != filepath.Clean(first) {
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
}
}
func TestResolveSessionConfigPathWithCandidatesMissing(t *testing.T) {
_, err := resolveSessionConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "no default session config found") {
t.Fatalf("error = %q, want missing-defaults context", err.Error())
}
if !strings.Contains(err.Error(), "pass --session") {
t.Fatalf("error = %q, want explicit-path guidance", err.Error())
}
}

View File

@@ -1,6 +1,10 @@
package artifacts
import "path/filepath"
import (
"path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// SessionPaths contains canonical local paths for one session work directory.
type SessionPaths struct {
@@ -20,24 +24,33 @@ type SessionPaths struct {
// SessionWorkDir returns the work directory for one session.
func SessionWorkDir(rootDir, sessionID string) string {
return filepath.Join(rootDir, "work", sessionID)
return filepath.Join(rootDir, config.PathWorkDirSegment, sessionID)
}
// SessionRunWorkDir returns the campaign/session/run scoped local work directory.
func SessionRunWorkDir(rootDir, campaign, sessionID, runID string) string {
return filepath.Join(rootDir, config.PathWorkDirSegment, campaign, sessionID, runID)
}
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
}
func buildSessionPaths(workspaceRoot, sessionID string) SessionPaths {
root := SessionWorkDir(workspaceRoot, sessionID)
transcripts := filepath.Join(root, "transcripts")
return SessionPaths{
WorkspaceRoot: workspaceRoot,
Root: root,
InputsDir: filepath.Join(root, "inputs"),
AudioDir: filepath.Join(root, "audio"),
TranscriptsDir: transcripts,
TranscriptsRawDir: filepath.Join(transcripts, "raw"),
TranscriptsTrimmedDir: filepath.Join(transcripts, "trimmed"),
ArtifactsDir: filepath.Join(root, "artifacts"),
ConfigDir: filepath.Join(root, "config"),
LogsDir: filepath.Join(root, "logs"),
ManifestPath: filepath.Join(root, "manifest.json"),
LockPath: filepath.Join(root, ".lock"),
InputsDir: filepath.Join(root, config.PathInputsDirSegment),
AudioDir: filepath.Join(root, config.PathAudioDirSegment),
TranscriptsDir: filepath.Join(root, config.PathTranscriptsSegment),
TranscriptsRawDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsRaw)),
TranscriptsTrimmedDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsTrimmed)),
ArtifactsDir: filepath.Join(root, config.PathArtifactsDirSegment),
ConfigDir: filepath.Join(root, config.PathConfigDirSegment),
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
ManifestPath: filepath.Join(root, config.PathManifestFile),
LockPath: filepath.Join(root, config.PathLockFile),
}
}

View File

@@ -0,0 +1,24 @@
package artifacts
import (
"path/filepath"
"testing"
)
func TestSessionRunWorkDir(t *testing.T) {
root := "/tmp/workspace"
got := SessionRunWorkDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
if got != want {
t.Fatalf("SessionRunWorkDir() = %q, want %q", got, want)
}
}
func TestSessionSpoolAudioDir(t *testing.T) {
root := "/var/spool/narratio"
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
want := filepath.Join(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4", "audio")
if got != want {
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
}
}

View File

@@ -0,0 +1,30 @@
package artifacts
import (
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"time"
)
// NewRunID returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx.
func NewRunID() (string, error) {
return NewRunIDWith(time.Now().UTC(), rand.Reader)
}
// NewRunIDWith returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx
// using an injected timestamp and randomness source.
func NewRunIDWith(now time.Time, random io.Reader) (string, error) {
if random == nil {
random = rand.Reader
}
var suffix [4]byte
if _, err := io.ReadFull(random, suffix[:]); err != nil {
return "", fmt.Errorf("generate run id random suffix: %w", err)
}
ts := now.UTC().Format("20060102T150405Z")
return ts + "-" + hex.EncodeToString(suffix[:]), nil
}

View File

@@ -0,0 +1,37 @@
package artifacts
import (
"bytes"
"regexp"
"strings"
"testing"
"time"
)
func TestNewRunIDWithFormat(t *testing.T) {
now := time.Date(2026, 5, 15, 3, 15, 22, 0, time.UTC)
random := bytes.NewReader([]byte{0xa1, 0xb2, 0xc3, 0xd4})
runID, err := NewRunIDWith(now, random)
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
if runID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("runID = %q, want %q", runID, "20260515T031522Z-a1b2c3d4")
}
}
func TestNewRunIDWithShape(t *testing.T) {
runID, err := NewRunIDWith(time.Now().UTC(), bytes.NewReader([]byte{0x01, 0x02, 0x03, 0x04}))
if err != nil {
t.Fatalf("NewRunIDWith() error = %v", err)
}
pattern := regexp.MustCompile(`^\d{8}T\d{6}Z-[0-9a-f]{8}$`)
if !pattern.MatchString(runID) {
t.Fatalf("runID = %q, want pattern %q", runID, pattern.String())
}
suffix := runID[len(runID)-8:]
if strings.ToLower(suffix) != suffix {
t.Fatalf("runID suffix = %q, want lowercase", suffix)
}
}

View File

@@ -0,0 +1,75 @@
package artifacts
import (
"path"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
// S3SessionPrefix builds the canonical S3 session prefix.
// Format: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
func S3SessionPrefix(rootPrefix, campaign, sessionID string) string {
prefix := path.Join(
cleanS3PathPart(rootPrefix),
config.S3CampaignsSegment,
cleanS3PathPart(campaign),
config.S3SessionsSegment,
cleanS3PathPart(sessionID),
)
return ensureS3TrailingSlash(prefix)
}
// S3RunPrefix builds the canonical S3 run prefix.
// Format: {session_prefix}/runs/{run_id}/
func S3RunPrefix(sessionPrefix, runID string) string {
prefix := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3RunsSegment, cleanS3PathPart(runID))
return ensureS3TrailingSlash(prefix)
}
// S3AudioPrefix builds the session audio prefix from configured audio_s3.prefix.
// Format: {session_prefix}/{audio_s3.prefix}
func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
key := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(audioPrefix))
return ensureS3TrailingSlash(key)
}
// S3CurrentManifestKey returns the current manifest pointer key.
// Format: {session_prefix}/current/manifest.json
func S3CurrentManifestKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3ManifestFile)
}
// S3CurrentRunPointerKey returns the current run pointer key.
// Format: {session_prefix}/current/run_id.txt
func S3CurrentRunPointerKey(sessionPrefix string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3RunIDFile)
}
// S3PromotedArtifactKey returns the destination key for one promoted artifact.
// Format: {session_prefix}/{promotion.to}
func S3PromotedArtifactKey(sessionPrefix, to string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
}
// S3RunRelativeDestinationKey returns a run-scoped key for a workdir-relative path.
// Format: {run_prefix}/{relative_workdir_path}
func S3RunRelativeDestinationKey(runPrefix, relativeWorkdirPath string) string {
return path.Join(strings.TrimSuffix(cleanS3Key(runPrefix), "/"), cleanS3Key(relativeWorkdirPath))
}
func ensureS3TrailingSlash(v string) string {
key := cleanS3Key(v)
if key == "" {
return ""
}
return strings.TrimSuffix(key, "/") + "/"
}
func cleanS3PathPart(v string) string {
return strings.Trim(strings.ReplaceAll(strings.TrimSpace(v), "\\", "/"), "/")
}
func cleanS3Key(v string) string {
return strings.ReplaceAll(strings.TrimSpace(v), "\\", "/")
}

View File

@@ -0,0 +1,45 @@
package artifacts
import (
"strings"
"testing"
)
func TestS3KeyConstruction(t *testing.T) {
runID := "20260515T031522Z-a1b2c3d4"
sessionPrefix := S3SessionPrefix("dnd", "forsaken", "2026-04-19")
if sessionPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/" {
t.Fatalf("sessionPrefix = %q", sessionPrefix)
}
audioPrefix := S3AudioPrefix(sessionPrefix, "audio/")
if audioPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/audio/" {
t.Fatalf("audioPrefix = %q", audioPrefix)
}
runPrefix := S3RunPrefix(sessionPrefix, runID)
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
if runPrefix != wantRunPrefix {
t.Fatalf("runPrefix = %q, want %q", runPrefix, wantRunPrefix)
}
runPointer := S3CurrentRunPointerKey(sessionPrefix)
if runPointer != "dnd/campaigns/forsaken/sessions/2026-04-19/current/run_id.txt" {
t.Fatalf("run pointer key = %q", runPointer)
}
manifestKey := S3CurrentManifestKey(sessionPrefix)
if manifestKey != "dnd/campaigns/forsaken/sessions/2026-04-19/current/manifest.json" {
t.Fatalf("manifest key = %q", manifestKey)
}
promoted := S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/trimmed.json" {
t.Fatalf("promoted key = %q", promoted)
}
runRelative := S3RunRelativeDestinationKey(runPrefix, `logs\whisperx.stdout.log`)
if !strings.HasSuffix(runRelative, "/logs/whisperx.stdout.log") {
t.Fatalf("runRelative key = %q, want normalized forward slashes", runRelative)
}
}

View File

@@ -12,6 +12,9 @@ type Config struct {
type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"`
Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"`
Archive *ArchiveConfig `yaml:"archive"`
Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim"`
Audita AuditaConfig `yaml:"audita"`
@@ -33,14 +36,52 @@ type SessionConfig struct {
// WorkspaceConfig configures local workspace behavior.
type WorkspaceConfig struct {
Root string `yaml:"root"`
Root string `yaml:"root"`
CleanupAfterArchive bool `yaml:"cleanup_after_archive"`
}
// SecretsConfig configures optional local filesystem secret loading.
type SecretsConfig struct {
EnvDir string `yaml:"env_dir"`
}
// StorageConfig configures storage backends and related parameters.
type StorageConfig struct {
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
Backend string `yaml:"backend"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
S3 *StorageS3Config `yaml:"s3"`
}
// StorageS3Config configures S3 storage coordinates.
type StorageS3Config struct {
Bucket string `yaml:"bucket"`
RootPrefix string `yaml:"root_prefix"`
Region string `yaml:"region"`
Endpoint string `yaml:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style"`
AccessKeyIDEnv string `yaml:"access_key_id_env"`
SecretKeyEnv string `yaml:"secret_access_key_env"`
}
// SpoolConfig configures local spool storage for staged data.
type SpoolConfig struct {
Root string `yaml:"root"`
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
}
// ArchiveConfig configures archive behavior and artifact promotions.
type ArchiveConfig struct {
Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"`
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
}
// ArchivePromotionRule configures one artifact promotion mapping.
type ArchivePromotionRule struct {
From string `yaml:"from"`
To string `yaml:"to"`
Required *bool `yaml:"required"`
}
// WhisperXConfig configures WhisperX adapter settings.
@@ -79,9 +120,14 @@ type AuditaConfig struct {
Modules []string `yaml:"modules"`
BaseURL string `yaml:"base_url"`
Model string `yaml:"model"`
LLMConcurrency *int `yaml:"llm_concurrency"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description"`
ConfigPath string `yaml:"config_path"`
OutputSchema string `yaml:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention"`
Report *bool `yaml:"report"`
}
@@ -169,9 +215,15 @@ type ArtifactSettings struct {
// SessionInputsConfig contains per-session input references.
type SessionInputsConfig struct {
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
}
// SessionAudioS3Input configures S3 session-audio input discovery.
type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"`
}

View File

@@ -0,0 +1,95 @@
package config
// Default filesystem locations for pipeline configuration lookup when --config
// is omitted. Order is highest to lowest precedence.
const (
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
DefaultSessionConfigPathLocal = "./session.yml"
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
DefaultS3SecretAccessKeyEnv = "OBJECT_STORAGE_KEY"
DefaultStorageS3RootPrefix = "dnd"
DefaultSpoolRoot = "/var/spool/narratio"
DefaultWhisperXLanguage = "en"
DefaultWhisperXTimeout = "30m"
DefaultWhisperXRetryDelay = "2s"
DefaultWhisperXConcurrency = 2
DefaultWhisperXRetries = 3
DefaultSeriatimBinary = "seriatim"
DefaultSeriatimTimeout = "10m"
DefaultSeriatimOutputSchema = "seriatim-intermediate"
DefaultSeriatimCoalesceGap = 3.0
DefaultSeriatimReport = true
DefaultAuditaBinary = "audita"
DefaultAuditaTimeout = "3h"
DefaultAuditaReport = true
DefaultScriptoriumBinary = "scriptorium"
DefaultScriptoriumTimeout = "10m"
DefaultTrimBoundsTimeout = "10m"
DefaultTrimSeriatimReport = false
DefaultNormalizeOutputPath = "transcripts/normalized.json"
DefaultNormalizeOutputSchema = "seriatim-intermediate"
DefaultNormalizeReport = true
DefaultArchiveEnabled = true
DefaultArchiveUploadRun = true
PathWorkDirSegment = "work"
PathInputsDirSegment = "inputs"
PathAudioDirSegment = "audio"
PathTranscriptsSegment = "transcripts"
PathTranscriptsRaw = "transcripts/raw"
PathTranscriptsTrimmed = "transcripts/trimmed"
PathArtifactsDirSegment = "artifacts"
PathConfigDirSegment = "config"
PathLogsDirSegment = "logs"
PathManifestFile = "manifest.json"
PathLockFile = ".lock"
PathTranscriptMerged = "transcripts/merged.json"
PathTranscriptProcessed = "transcripts/processed.json"
PathTranscriptNormalized = "transcripts/normalized.json"
PathTranscriptTrimmed = "transcripts/trimmed.json"
S3CampaignsSegment = "campaigns"
S3SessionsSegment = "sessions"
S3RunsSegment = "runs"
S3CurrentSegment = "current"
S3ManifestFile = "manifest.json"
S3RunIDFile = "run_id.txt"
)
// DefaultArchivePromoteArtifacts defines the default archive promotion rules.
// Callers should copy this slice before mutating.
var DefaultArchivePromoteArtifacts = []ArchivePromotionRule{
{From: PathTranscriptTrimmed, To: PathTranscriptTrimmed},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md"},
}
// DefaultPipelineConfigSearchPaths defines the default search order for
// pipeline.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultPipelineConfigSearchPaths = []string{
DefaultPipelineConfigPathUsrLocal,
DefaultPipelineConfigPathEtc,
}
// DefaultSessionConfigSearchPaths defines the default search order for
// session.yml when callers do not provide an explicit path.
//
// Keep this in a variable so future defaults can be extended without changing
// call sites.
var DefaultSessionConfigSearchPaths = []string{
DefaultSessionConfigPathLocal,
DefaultSessionConfigPathUsrLocal,
DefaultSessionConfigPathEtc,
}

View File

@@ -5,6 +5,8 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
@@ -21,21 +23,56 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
// LoadSession loads session configuration from a YAML file with strict field checking.
func LoadSession(path string) (*SessionConfig, error) {
var cfg SessionConfig
if err := decodeStrictYAML("session", path, &cfg); err != nil {
return LoadSessionWithOptions(path, SessionLoadOptions{})
}
// SessionLoadOptions configures session template rendering behavior.
type SessionLoadOptions struct {
SessionID string
}
// LoadSessionWithOptions loads session configuration from a YAML file with
// strict field checking after template rendering.
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
sessionBytes, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
}
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
if err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
var cfg SessionConfig
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
path,
strings.TrimSpace(opts.SessionID),
strings.TrimSpace(cfg.SessionID),
)
}
return &cfg, nil
}
// Load loads and resolves combined pipeline and session configuration.
func Load(pipelinePath, sessionPath string) (*Config, error) {
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
}
// LoadWithSessionOptions loads and resolves combined pipeline and session
// configuration with session template options.
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipeline(pipelinePath)
if err != nil {
return nil, err
}
sessionCfg, err := LoadSession(sessionPath)
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
if err != nil {
return nil, err
}
@@ -55,7 +92,11 @@ func decodeStrictYAML(kind, path string, out any) error {
}
defer f.Close()
dec := yaml.NewDecoder(f)
return decodeStrictYAMLFromReader(kind, path, f, out)
}
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
dec := yaml.NewDecoder(r)
dec.KnownFields(true)
if err := dec.Decode(out); err != nil {
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
@@ -69,6 +110,36 @@ func decodeStrictYAML(kind, path string, out any) error {
return nil
}
var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
sessionID := strings.TrimSpace(opts.SessionID)
rendered := content
if sessionID != "" {
rendered = strings.ReplaceAll(rendered, "{{session_id}}", sessionID)
rendered = strings.ReplaceAll(rendered, "{{ session_id }}", sessionID)
}
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
if len(unresolved) > 0 {
vars := make([]string, 0, len(unresolved))
for _, m := range unresolved {
if len(m) > 1 {
vars = append(vars, m[1])
}
}
if len(vars) > 0 {
return "", fmt.Errorf(
"session file template rendering failed: unresolved template variable(s): %s; pass --session-id when using {{ session_id }}",
strings.Join(vars, ", "),
)
}
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
}
return rendered, nil
}
func shortName(path, fallback string) string {
base := filepath.Base(path)
if base == "." || base == string(filepath.Separator) {
@@ -81,6 +152,9 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyArchiveDefaults(&cfg.Archive)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
@@ -92,24 +166,75 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
applyScriptoriumDefaults(cfg.Scriptorium)
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
if cfg.S3 == nil {
cfg.S3 = &StorageS3Config{}
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyArchiveDefaults(cfg **ArchiveConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &ArchiveConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).PromoteArtifacts) == 0 {
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
}
for i := range (*cfg).PromoteArtifacts {
if (*cfg).PromoteArtifacts[i].Required == nil {
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = "en"
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = "30m"
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = "2s"
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(2)
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(3)
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
@@ -122,17 +247,20 @@ func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "10m"
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = "seriatim-intermediate"
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(3.0)
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
@@ -140,34 +268,14 @@ func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "3h"
}
if cfg.Modules == nil {
cfg.Modules = []string{
"glossary",
"homophones",
"glossary",
"spoken_word",
"grammar",
"homophones",
"glossary",
}
}
if cfg.BaseURL == "" {
cfg.BaseURL = "https://openrouter.ai/api/v1"
}
if cfg.Model == "" {
cfg.Model = "openrouter/google/gemma-4-31b-it"
}
if cfg.LLMConcurrency == nil {
cfg.LLMConcurrency = intPtr(1)
}
if cfg.ValidationLLMConcurrency == nil {
cfg.ValidationLLMConcurrency = intPtr(1)
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
@@ -175,8 +283,11 @@ func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = "10m"
cfg.Timeout = DefaultScriptoriumTimeout
}
}
@@ -185,10 +296,10 @@ func applyTrimDefaults(cfg *TrimConfig) {
return
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = "10m"
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(false)
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
@@ -197,13 +308,13 @@ func applyNormalizeDefaults(cfg *NormalizeConfig) {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = defaultNormalizeOutputSchema
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = defaultNormalizeOutputPath
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(true)
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}

View File

@@ -37,6 +37,26 @@ inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
checkDefault: true,
},
{
name: "seriatim and audita sections can be omitted",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
analyzer:
timeout: 20m
notification:
timeout: 15s
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
checkDefault: true,
},
@@ -72,6 +92,46 @@ inputs:
`,
wantLoadErr: "strict decode failed",
},
{
name: "unknown secrets field fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
secrets:
bogus: true
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantLoadErr: "strict decode failed",
},
{
name: "empty secrets env_dir fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
secrets:
env_dir: " "
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured",
},
{
name: "unknown session field fails",
pipelineYAML: `workspace:
@@ -256,7 +316,7 @@ inputs:
wantLoadErr: "strict decode failed",
},
{
name: "missing seriatim binary fails",
name: "missing seriatim binary uses default",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -271,7 +331,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.seriatim.binary is required",
},
{
name: "invalid seriatim timeout fails",
@@ -372,7 +431,7 @@ inputs:
wantLoadErr: "strict decode failed",
},
{
name: "missing audita binary fails",
name: "missing audita binary uses default",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -389,7 +448,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.binary is required",
},
{
name: "invalid audita timeout fails",
@@ -413,7 +471,7 @@ inputs:
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.timeout must be a valid duration",
},
{
name: "empty audita modules fails",
name: "empty audita modules is valid override",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -431,7 +489,6 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.modules must include at least one module",
},
{
name: "empty audita module item fails",
@@ -501,7 +558,7 @@ inputs:
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.base_url must be a valid URL",
},
{
name: "invalid audita llm_concurrency fails",
name: "legacy audita llm_concurrency field fails strict decode",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
@@ -510,7 +567,7 @@ seriatim:
binary: seriatim
audita:
binary: audita
llm_concurrency: 0
llm_concurrency: 1
`,
sessionYAML: `session_id: 2026-05-03
inputs:
@@ -519,7 +576,49 @@ inputs:
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.llm_concurrency must be > 0",
wantLoadErr: "strict decode failed",
},
{
name: "invalid audita total_llm_concurrency fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
total_llm_concurrency: 0
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.total_llm_concurrency must be > 0",
},
{
name: "invalid audita proposal_llm_concurrency fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
proposal_llm_concurrency: 0
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.proposal_llm_concurrency must be > 0",
},
{
name: "invalid audita validation_llm_concurrency fails",
@@ -542,6 +641,48 @@ inputs:
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.validation_llm_concurrency must be > 0",
},
{
name: "invalid audita output_schema fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
output_schema: bad
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.output_schema must be one of: bare-segments, audita-v1",
},
{
name: "invalid audita work_dir_retention fails",
pipelineYAML: `workspace:
root: /tmp/narratio
whisperx:
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
seriatim:
binary: seriatim
audita:
binary: audita
work_dir_retention: sometimes
`,
sessionYAML: `session_id: 2026-05-03
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.work_dir_retention must be one of: always, auto, never",
},
}
for _, tt := range tests {
@@ -590,6 +731,9 @@ inputs:
if cfg.Pipeline.Seriatim.Timeout != "10m" {
t.Fatalf("seriatim.timeout = %q, want %q", cfg.Pipeline.Seriatim.Timeout, "10m")
}
if cfg.Pipeline.Seriatim.Binary != "seriatim" {
t.Fatalf("seriatim.binary = %q, want %q", cfg.Pipeline.Seriatim.Binary, "seriatim")
}
if cfg.Pipeline.Seriatim.OutputSchema != "seriatim-intermediate" {
t.Fatalf("seriatim.output_schema = %q, want %q", cfg.Pipeline.Seriatim.OutputSchema, "seriatim-intermediate")
}
@@ -602,26 +746,32 @@ inputs:
if cfg.Pipeline.Audita.Timeout != "3h" {
t.Fatalf("audita.timeout = %q, want %q", cfg.Pipeline.Audita.Timeout, "3h")
}
if cfg.Pipeline.Audita.Binary != "audita" {
t.Fatalf("audita.binary = %q, want %q", cfg.Pipeline.Audita.Binary, "audita")
}
if cfg.Pipeline.Audita.LLMAPIKeyEnv != "" {
t.Fatalf("audita.llm_api_key_env = %q, want empty by default", cfg.Pipeline.Audita.LLMAPIKeyEnv)
}
if got := strings.Join(cfg.Pipeline.Audita.Modules, ","); got != "glossary,homophones,glossary,spoken_word,grammar,homophones,glossary" {
t.Fatalf("audita.modules = %q, want default sequence", got)
if cfg.Pipeline.Audita.Modules != nil {
t.Fatalf("audita.modules = %#v, want nil default (optional override)", cfg.Pipeline.Audita.Modules)
}
if cfg.Pipeline.Audita.BaseURL != "https://openrouter.ai/api/v1" {
t.Fatalf("audita.base_url = %q, want %q", cfg.Pipeline.Audita.BaseURL, "https://openrouter.ai/api/v1")
if cfg.Pipeline.Audita.BaseURL != "" {
t.Fatalf("audita.base_url = %q, want empty default", cfg.Pipeline.Audita.BaseURL)
}
if cfg.Pipeline.Audita.Model != "openrouter/google/gemma-4-31b-it" {
t.Fatalf("audita.model = %q, want %q", cfg.Pipeline.Audita.Model, "openrouter/google/gemma-4-31b-it")
}
if cfg.Pipeline.Audita.LLMConcurrency == nil || *cfg.Pipeline.Audita.LLMConcurrency != 1 {
t.Fatalf("audita.llm_concurrency = %v, want 1", cfg.Pipeline.Audita.LLMConcurrency)
if cfg.Pipeline.Audita.Model != "" {
t.Fatalf("audita.model = %q, want empty default", cfg.Pipeline.Audita.Model)
}
if cfg.Pipeline.Audita.ValidationModel != "" {
t.Fatalf("audita.validation_model = %q, want empty default", cfg.Pipeline.Audita.ValidationModel)
}
if cfg.Pipeline.Audita.ValidationLLMConcurrency == nil || *cfg.Pipeline.Audita.ValidationLLMConcurrency != 1 {
t.Fatalf("audita.validation_llm_concurrency = %v, want 1", cfg.Pipeline.Audita.ValidationLLMConcurrency)
if cfg.Pipeline.Audita.TotalLLMConcurrency != nil {
t.Fatalf("audita.total_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.TotalLLMConcurrency)
}
if cfg.Pipeline.Audita.ProposalLLMConcurrency != nil {
t.Fatalf("audita.proposal_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ProposalLLMConcurrency)
}
if cfg.Pipeline.Audita.ValidationLLMConcurrency != nil {
t.Fatalf("audita.validation_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ValidationLLMConcurrency)
}
if cfg.Pipeline.Audita.Report == nil || *cfg.Pipeline.Audita.Report != true {
t.Fatalf("audita.report = %v, want true", cfg.Pipeline.Audita.Report)
@@ -685,7 +835,8 @@ func TestValidateMissingAudioSource(t *testing.T) {
Modules: []string{"glossary", "homophones"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: intPtr(1),
TotalLLMConcurrency: intPtr(1),
ProposalLLMConcurrency: intPtr(1),
ValidationModel: "",
ValidationLLMConcurrency: intPtr(1),
Report: boolPtr(true),
@@ -693,6 +844,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
},
Session: &SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: SessionInputsConfig{
SpeakersFile: "speakers.yml",
AutocorrectFile: "autocorrect.yml",
@@ -705,7 +857,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
if err == nil {
t.Fatal("expected validation error, got nil")
}
if !strings.Contains(err.Error(), "audio_dir or at least one audio_files") {
if !strings.Contains(err.Error(), "audio_dir, at least one audio_files entry, or audio_s3") {
t.Fatalf("error = %q, want audio source guidance", err.Error())
}
if !strings.Contains(err.Error(), "session config") {
@@ -734,6 +886,12 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
}
pipelineYAML += "audita:\n binary: audita\n"
}
if !strings.Contains(sessionYAML, "\ncampaign:") && !strings.HasPrefix(sessionYAML, "campaign:") {
if !strings.HasSuffix(sessionYAML, "\n") {
sessionYAML += "\n"
}
sessionYAML += "campaign: sample-campaign\n"
}
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")

View File

@@ -6,11 +6,6 @@ import (
"gopkg.in/yaml.v3"
)
const (
defaultNormalizeOutputPath = "transcripts/normalized.json"
defaultNormalizeOutputSchema = "seriatim-intermediate"
)
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
// distinguish omitted vs explicitly empty values.
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {

View File

@@ -49,11 +49,19 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
wantLoadErr: "strict decode failed",
},
{
name: "missing binary fails when section present",
name: "missing binary defaults when section present",
scriptoriumYAML: `scriptorium:
timeout: 10m
`,
wantValidateErr: "pipeline.scriptorium.binary is required",
assert: func(t *testing.T, cfg *Config) {
t.Helper()
if cfg.Pipeline.Scriptorium == nil {
t.Fatal("scriptorium config should be present")
}
if cfg.Pipeline.Scriptorium.Binary != "scriptorium" {
t.Fatalf("scriptorium.binary = %q, want scriptorium", cfg.Pipeline.Scriptorium.Binary)
}
},
},
{
name: "enabled artifact missing prompt id fails",

View File

@@ -0,0 +1,156 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestLoadSessionWithOptionsRendersCompactPlaceholder(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{session_id}}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-04-04" {
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
}
}
func TestLoadSessionWithOptionsRendersSpacedPlaceholder(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-04-04" {
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
}
}
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "unresolved template variable") {
t.Fatalf("error = %q, want unresolved-variable context", err.Error())
}
if !strings.Contains(err.Error(), "session_id") {
t.Fatalf("error = %q, want session_id variable", err.Error())
}
}
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "session_id mismatch") {
t.Fatalf("error = %q, want mismatch context", err.Error())
}
}
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: "{{ session_id }}"
campaign: sample-campaign
unknown_field: true
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("error = %q, want strict-decode context", err.Error())
}
}
func TestLoadSessionWithOptionsConcreteSessionStillLoads(t *testing.T) {
dir := t.TempDir()
sessionPath := filepath.Join(dir, "session.yml")
sessionYAML := `session_id: 2026-05-03
campaign: sample-campaign
inputs:
audio_dir: ./audio
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
t.Fatalf("write session.yml: %v", err)
}
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
if err != nil {
t.Fatalf("LoadSessionWithOptions() error = %v", err)
}
if cfg.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
}
}

View File

@@ -0,0 +1,315 @@
package config
import (
"strings"
"testing"
)
func TestStorageS3DefaultsAndValidation(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3 == nil {
t.Fatal("storage.s3 should be initialized")
}
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != DefaultS3AccessKeyIDEnv {
t.Fatalf("storage.s3.access_key_id_env = %q, want %q", cfg.Pipeline.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != DefaultS3SecretAccessKeyEnv {
t.Fatalf("storage.s3.secret_access_key_env = %q, want %q", cfg.Pipeline.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv)
}
if cfg.Pipeline.Storage.S3.ForcePathStyle {
t.Fatalf("storage.s3.force_path_style = true, want false default")
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvNamesLoadAndValidate(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: CUSTOM_KEY_ID
secret_access_key_env: CUSTOM_SECRET
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != "CUSTOM_KEY_ID" {
t.Fatalf("storage.s3.access_key_id_env = %q, want CUSTOM_KEY_ID", cfg.Pipeline.Storage.S3.AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != "CUSTOM_SECRET" {
t.Fatalf("storage.s3.secret_access_key_env = %q, want CUSTOM_SECRET", cfg.Pipeline.Storage.S3.SecretKeyEnv)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "invalid access key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: "123BAD"
`,
wantErr: "pipeline.storage.s3.access_key_id_env must be a valid environment variable name",
},
{
name: "invalid secret key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
secret_access_key_env: "bad-name"
`,
wantErr: "pipeline.storage.s3.secret_access_key_env must be a valid environment variable name",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSpoolAndArchiveDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
}
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
t.Fatalf("spool.delete_audio_after_archive = true, want false")
}
if cfg.Pipeline.Workspace.CleanupAfterArchive {
t.Fatalf("workspace.cleanup_after_archive = true, want false")
}
if cfg.Pipeline.Archive == nil {
t.Fatal("archive should be initialized by defaults")
}
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
}
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
}
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 2 {
t.Fatalf("archive.promote_artifacts len = %d, want 2 defaults", len(cfg.Pipeline.Archive.PromoteArtifacts))
}
for i, item := range cfg.Pipeline.Archive.PromoteArtifacts {
if item.Required == nil || !*item.Required {
t.Fatalf("archive.promote_artifacts[%d].required = %#v, want true", i, item.Required)
}
}
}
func TestArchivePromotionPathValidation(t *testing.T) {
tests := []struct {
name string
ruleYML string
wantErr string
}{
{
name: "absolute from path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "/transcripts/trimmed.json"
to: "transcripts/trimmed.json"
`,
wantErr: "must be a relative path",
},
{
name: "traversal to path rejected",
ruleYML: `archive:
promote_artifacts:
- from: "transcripts/trimmed.json"
to: "../trimmed.json"
`,
wantErr: "must not contain path traversal",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + "\n" + tt.ruleYML
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct {
name string
sessionYAML string
wantErr string
}{
{
name: "valid audio_s3 prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
},
{
name: "invalid audio_s3 absolute prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: /audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must be a relative path",
},
{
name: "invalid audio_s3 traversal prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: ../audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must not contain path traversal",
},
{
name: "local and s3 audio conflict",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_dir: ./audio
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "mutually exclusive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, tt.sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestStorageS3BucketRequiredWhenS3DependentFeatureEnabled(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
`
sessionYAML := `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3.bucket is required") {
t.Fatalf("Validate() error = %v, want bucket requirement", err)
}
}
func TestLocalAudioConfigStillValid(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}

View File

@@ -3,6 +3,8 @@ package config
import (
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
)
@@ -25,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
}
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
return fmt.Errorf("pipeline/session config invalid: %w", err)
}
return nil
}
@@ -33,6 +38,18 @@ func validatePipeline(cfg *PipelineConfig) error {
if strings.TrimSpace(cfg.Workspace.Root) == "" {
return fmt.Errorf("pipeline.workspace.root is required")
}
if err := validateSecrets(cfg.Secrets); err != nil {
return err
}
if err := validateStorage(cfg.Storage); err != nil {
return err
}
if err := validateSpool(cfg.Spool); err != nil {
return err
}
if err := validateArchive(cfg.Archive); err != nil {
return err
}
if err := validateWhisperX(cfg.WhisperX); err != nil {
return err
}
@@ -61,6 +78,64 @@ func validatePipeline(cfg *PipelineConfig) error {
return nil
}
func validateStorage(cfg StorageConfig) error {
if cfg.S3 == nil {
return nil
}
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
}
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
return err
}
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
}
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
return err
}
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
return err
}
return nil
}
func validateSpool(cfg SpoolConfig) error {
return nil
}
func validateArchive(cfg *ArchiveConfig) error {
if cfg == nil {
return nil
}
for i, item := range cfg.PromoteArtifacts {
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
if strings.TrimSpace(item.From) == "" {
return fmt.Errorf("%s.from is required", prefix)
}
if strings.TrimSpace(item.To) == "" {
return fmt.Errorf("%s.to is required", prefix)
}
if err := validateRelativeSafePath(prefix+".from", item.From); err != nil {
return err
}
if err := validateRelativeSafePath(prefix+".to", item.To); err != nil {
return err
}
}
return nil
}
func validateSecrets(cfg *SecretsConfig) error {
if cfg == nil {
return nil
}
if strings.TrimSpace(cfg.EnvDir) == "" {
return fmt.Errorf("pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured")
}
return nil
}
func validateNormalize(cfg *NormalizeConfig) error {
if cfg == nil {
return nil
@@ -187,15 +262,12 @@ func validateAudita(cfg AuditaConfig) error {
if err := validateDuration("pipeline.audita.timeout", cfg.Timeout); err != nil {
return err
}
if len(cfg.Modules) == 0 {
return fmt.Errorf("pipeline.audita.modules must include at least one module")
}
for i, mod := range cfg.Modules {
m := strings.TrimSpace(mod)
if m == "" {
for i, m := range cfg.Modules {
module := strings.TrimSpace(m)
if module == "" {
return fmt.Errorf("pipeline.audita.modules[%d] must be non-empty", i)
}
switch m {
switch module {
case "glossary", "homophones", "spoken_word", "grammar":
default:
return fmt.Errorf("pipeline.audita.modules[%d] must be one of: glossary, homophones, spoken_word, grammar", i)
@@ -210,21 +282,31 @@ func validateAudita(cfg AuditaConfig) error {
return fmt.Errorf("pipeline.audita.base_url must be a valid URL")
}
}
if strings.TrimSpace(cfg.Model) == "" {
return fmt.Errorf("pipeline.audita.model is required")
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.total_llm_concurrency must be > 0")
}
if cfg.LLMConcurrency == nil {
return fmt.Errorf("pipeline.audita.llm_concurrency must be set (defaults should populate this)")
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.proposal_llm_concurrency must be > 0")
}
if *cfg.LLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.llm_concurrency must be > 0")
}
if cfg.ValidationLLMConcurrency == nil {
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be set (defaults should populate this)")
}
if *cfg.ValidationLLMConcurrency <= 0 {
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be > 0")
}
if strings.TrimSpace(cfg.TranscriptDescription) == "" && cfg.TranscriptDescription != "" {
return fmt.Errorf("pipeline.audita.transcript_description must be non-empty when provided")
}
if strings.TrimSpace(cfg.ConfigPath) == "" && cfg.ConfigPath != "" {
return fmt.Errorf("pipeline.audita.config_path must be non-empty when provided")
}
switch strings.TrimSpace(cfg.OutputSchema) {
case "", "bare-segments", "audita-v1":
default:
return fmt.Errorf("pipeline.audita.output_schema must be one of: bare-segments, audita-v1")
}
switch strings.TrimSpace(cfg.WorkDirRetention) {
case "", "always", "auto", "never":
default:
return fmt.Errorf("pipeline.audita.work_dir_retention must be one of: always, auto, never")
}
return nil
}
@@ -284,6 +366,9 @@ func validateSession(cfg *SessionConfig) error {
if strings.TrimSpace(cfg.SessionID) == "" {
return fmt.Errorf("session.session_id is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("session.campaign is required")
}
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
return fmt.Errorf("session.inputs.speakers_file is required")
@@ -297,13 +382,91 @@ func validateSession(cfg *SessionConfig) error {
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
if !hasAudioDir && !hasAudioFiles {
return fmt.Errorf("session.inputs requires audio_dir or at least one audio_files entry")
hasAudioS3 := cfg.Inputs.AudioS3 != nil
if hasAudioS3 {
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
}
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
return err
}
}
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
}
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
}
return nil
}
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
if pipeline == nil || session == nil {
return nil
}
if pipeline.Storage.S3 == nil {
return nil
}
audioS3Enabled := session.Inputs.AudioS3 != nil
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
}
return nil
}
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
if pipeline == nil || pipeline.Archive == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
return false
}
enabled := true
if pipeline.Archive.Enabled != nil {
enabled = *pipeline.Archive.Enabled
}
upload := true
if pipeline.Archive.UploadRun != nil {
upload = *pipeline.Archive.UploadRun
}
return enabled && upload
}
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
func validateEnvVarNameField(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if !envVarNameRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a valid environment variable name", fieldName)
}
return nil
}
func validateRelativeSafePath(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a relative path", fieldName)
}
normalized := strings.ReplaceAll(trimmed, "\\", "/")
for _, segment := range strings.Split(normalized, "/") {
if segment == ".." {
return fmt.Errorf("%s must not contain path traversal", fieldName)
}
}
return nil
}
func validateDuration(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {

View File

@@ -14,9 +14,15 @@ type ErrorRecord struct {
// InputRecord captures one resolved input and optional checksum.
type InputRecord struct {
Kind string `json:"kind"`
Path string `json:"path"`
Checksum string `json:"checksum,omitempty"`
Kind string `json:"kind"`
Path string `json:"path"`
Checksum string `json:"checksum,omitempty"`
Source string `json:"source,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3Key string `json:"s3_key,omitempty"`
S3Size int64 `json:"s3_size,omitempty"`
S3ETag string `json:"s3_etag,omitempty"`
SpoolPath string `json:"spool_path,omitempty"`
}
// ArtifactRecord captures one produced artifact and optional remote metadata.
@@ -45,6 +51,13 @@ type StageRecord struct {
// Manifest is the durable run-state record for a session execution.
type Manifest struct {
SessionID string `json:"session_id"`
Campaign string `json:"campaign,omitempty"`
RunID string `json:"run_id,omitempty"`
LocalWorkDir string `json:"local_workdir,omitempty"`
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
PipelineVersion string `json:"pipeline_version,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`

View File

@@ -22,6 +22,13 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
now := time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC)
m.MarkStageRunning("prepare", now)
m.MarkStageSucceeded("prepare", now.Add(2*time.Second), []ArtifactRecord{{Kind: "transcript", LocalPath: "transcripts/merged.json"}})
m.Campaign = "forsaken"
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = "/var/lib/narratio/work/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4"
m.LocalSpoolDir = "/var/spool/narratio/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4/audio"
m.S3Bucket = "my-dnd-archive"
m.S3SessionPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/"
m.S3RunPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/runs/20260515T031522Z-a1b2c3d4/"
path := filepath.Join(t.TempDir(), "manifest.json")
if err := store.Save(ctx, path, m); err != nil {
@@ -36,6 +43,12 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
if loaded.SessionID != "2026-05-03" {
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
}
if loaded.Campaign != "forsaken" {
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
}
if loaded.RunID != "20260515T031522Z-a1b2c3d4" {
t.Fatalf("RunID = %q, want run id", loaded.RunID)
}
stage, ok := loaded.Stages["prepare"]
if !ok {
t.Fatalf("stage prepare not found")

612
internal/stage/archive.go Normal file
View File

@@ -0,0 +1,612 @@
package stage
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type archiveStage struct{}
type archiveUploadFile struct {
RelativePath string
LocalPath string
}
var archivePrerequisiteStages = []string{
"prepare",
"transcribe",
"merge",
"polish",
"normalize",
"trim",
"analyze",
}
var archiveRunUploadDirs = []string{
"inputs",
"transcripts",
"artifacts",
"reports",
"config",
"logs",
}
func (archiveStage) Name() string { return "archive" }
func (archiveStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "manifest", Category: "input", RelativePath: "manifest.json"},
},
}
}
func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return nil, fmt.Errorf("archive: resolved config must include pipeline and session")
}
if archiveDisabled(env) {
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"skipped": true,
"archive_enabled": false,
"audio_upload_skipped": true,
"current_pointer_written": false,
},
}, nil
}
if archiveRunUploadDisabled(env) {
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"skipped": true,
"upload_run_enabled": false,
"audio_upload_skipped": true,
"current_pointer_written": false,
},
}, nil
}
if err := validateArchivePrerequisites(m); err != nil {
return nil, fmt.Errorf("archive: %w", err)
}
if env.ObjectStore == nil {
return nil, fmt.Errorf("archive: remote object store backend is required when archive run upload is enabled")
}
workDir, err := archiveWorkDir(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve local workdir: %w", err)
}
workDirInfo, err := os.Stat(workDir)
if err != nil {
return nil, fmt.Errorf("archive: local workdir %q: %w", workDir, err)
}
if !workDirInfo.IsDir() {
return nil, fmt.Errorf("archive: local workdir %q is not a directory", workDir)
}
runPrefix, err := archiveRunPrefix(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve s3 run prefix: %w", err)
}
sessionPrefix, err := archiveSessionPrefix(env, m)
if err != nil {
return nil, fmt.Errorf("archive: resolve s3 session prefix: %w", err)
}
bucket := archiveBucket(env, m)
if bucket == "" {
return nil, fmt.Errorf("archive: resolve s3 bucket: bucket is required")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return nil, fmt.Errorf("archive: run id is required")
}
manifestSource, err := resolveArchiveManifestSource(env, m, workDir)
if err != nil {
return nil, fmt.Errorf("archive: resolve manifest source: %w", err)
}
runFiles, err := collectArchiveRunFiles(workDir, manifestSource)
if err != nil {
return nil, fmt.Errorf("archive: collect run files: %w", err)
}
workDirCandidates := archiveSourceWorkDirs(env, m, workDir)
promotions, err := resolveArchivePromotions(workDirCandidates, env.Config.Pipeline.Archive.PromoteArtifacts)
if err != nil {
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
}
runUploaded := make([]string, 0, len(runFiles))
for _, file := range runFiles {
key := artifacts.S3RunRelativeDestinationKey(runPrefix, file.RelativePath)
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
return nil, fmt.Errorf("archive: upload run file %q to %q: %w", file.RelativePath, key, err)
}
runUploaded = append(runUploaded, file.RelativePath)
}
promotedUploaded := make([]string, 0, len(promotions))
skippedOptional := make([]string, 0)
for _, promotion := range promotions {
if !promotion.Exists {
if promotion.Required {
return nil, fmt.Errorf("archive: required promotion source missing: %q", promotion.From)
}
skippedOptional = append(skippedOptional, promotion.To)
continue
}
key := artifacts.S3PromotedArtifactKey(sessionPrefix, promotion.To)
if _, err := env.ObjectStore.Upload(ctx, promotion.LocalPath, key, storage.UploadOptions{}); err != nil {
return nil, fmt.Errorf("archive: upload promoted output %q to %q: %w", promotion.From, key, err)
}
promotedUploaded = append(promotedUploaded, promotion.To)
}
currentManifestKey := artifacts.S3CurrentManifestKey(sessionPrefix)
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
bucket,
runPrefix,
sessionPrefix,
runUploaded,
promotedUploaded,
skippedOptional,
currentManifestKey,
))
if err != nil {
return nil, fmt.Errorf("archive: build current manifest snapshot: %w", err)
}
defer func() { _ = os.Remove(manifestTempPath) }()
if _, err := env.ObjectStore.Upload(ctx, manifestTempPath, currentManifestKey, storage.UploadOptions{
ContentType: "application/json",
}); err != nil {
return nil, fmt.Errorf("archive: upload current manifest to %q: %w", currentManifestKey, err)
}
currentRunPointerKey := artifacts.S3CurrentRunPointerKey(sessionPrefix)
runIDTempPath, err := writeCurrentRunIDPointer(runID)
if err != nil {
return nil, fmt.Errorf("archive: build current run id pointer: %w", err)
}
defer func() { _ = os.Remove(runIDTempPath) }()
if _, err := env.ObjectStore.Upload(ctx, runIDTempPath, currentRunPointerKey, storage.UploadOptions{
ContentType: "text/plain; charset=utf-8",
}); err != nil {
return nil, fmt.Errorf("archive: upload current run pointer to %q: %w", currentRunPointerKey, err)
}
return &StageResult{
Metadata: map[string]any{
"stage": "archive",
"uploaded": true,
"s3_bucket": bucket,
"s3_run_prefix": runPrefix,
"run_files_uploaded": len(runUploaded),
"run_uploaded_paths": runUploaded,
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": promotedUploaded,
"skipped_optional_promotions": skippedOptional,
"current_manifest_key": currentManifestKey,
"current_run_id_key": currentRunPointerKey,
"current_pointer_written": true,
"audio_upload_skipped": true,
},
}, nil
}
type archivePromotion struct {
From string
To string
Required bool
LocalPath string
Exists bool
}
func archiveDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
return true
}
return cfg.Enabled != nil && !*cfg.Enabled
}
func archiveRunUploadDisabled(env *Env) bool {
cfg := env.Config.Pipeline.Archive
if cfg == nil {
return true
}
return cfg.UploadRun != nil && !*cfg.UploadRun
}
func validateArchivePrerequisites(m *manifest.Manifest) error {
if m == nil {
return fmt.Errorf("manifest is required")
}
for _, stageName := range archivePrerequisiteStages {
sr := m.Stages[stageName]
if sr == nil {
return fmt.Errorf("prerequisite stage %q has not succeeded", stageName)
}
if sr.Status != manifest.StatusSucceeded {
return fmt.Errorf("prerequisite stage %q status is %q (want %q)", stageName, sr.Status, manifest.StatusSucceeded)
}
}
return nil
}
func archiveWorkDir(env *Env, m *manifest.Manifest) (string, error) {
workDir := strings.TrimSpace(m.LocalWorkDir)
if workDir != "" {
cleaned := filepath.Clean(workDir)
if info, err := os.Stat(cleaned); err == nil && info.IsDir() {
return cleaned, nil
}
}
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
campaign = strings.TrimSpace(m.Campaign)
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return "", fmt.Errorf("run id is required")
}
if campaign == "" || sessionID == "" {
return "", fmt.Errorf("campaign and session id are required")
}
runScoped := artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID)
if info, err := os.Stat(runScoped); err == nil && info.IsDir() {
return runScoped, nil
}
legacy := artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID)
return legacy, nil
}
func archiveRunPrefix(env *Env, m *manifest.Manifest) (string, error) {
runPrefix := strings.TrimSpace(m.S3RunPrefix)
if runPrefix != "" {
return runPrefix, nil
}
sessionPrefix, err := archiveSessionPrefix(env, m)
if err != nil {
return "", err
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return "", fmt.Errorf("run id is required")
}
return artifacts.S3RunPrefix(sessionPrefix, runID), nil
}
func archiveSessionPrefix(env *Env, m *manifest.Manifest) (string, error) {
if m != nil && strings.TrimSpace(m.S3SessionPrefix) != "" {
return strings.TrimSpace(m.S3SessionPrefix), nil
}
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" {
sessionID = strings.TrimSpace(m.SessionID)
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
campaign = strings.TrimSpace(m.Campaign)
}
if env.Config.Pipeline.Storage.S3 == nil {
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
if strings.TrimSpace(sessionPrefix) == "" {
return "", fmt.Errorf("session prefix is required")
}
return sessionPrefix, nil
}
func archiveBucket(env *Env, m *manifest.Manifest) string {
if m != nil && strings.TrimSpace(m.S3Bucket) != "" {
return strings.TrimSpace(m.S3Bucket)
}
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Pipeline.Storage.S3 == nil {
return ""
}
return strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
}
func resolveArchivePromotions(workDirs []string, rules []config.ArchivePromotionRule) ([]archivePromotion, error) {
if len(workDirs) == 0 {
return nil, fmt.Errorf("at least one workdir candidate is required")
}
out := make([]archivePromotion, 0, len(rules))
for _, rule := range rules {
from := strings.TrimSpace(rule.From)
to := strings.TrimSpace(rule.To)
required := rule.Required == nil || *rule.Required
var (
localPath string
exists bool
)
for _, candidateRoot := range workDirs {
resolvedPath, err := resolveWorkDirRelativePath(candidateRoot, from)
if err != nil {
return nil, fmt.Errorf("promotion from %q: %w", from, err)
}
info, err := os.Stat(resolvedPath)
if err == nil && !info.IsDir() {
localPath = resolvedPath
exists = true
break
}
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("promotion source %q: %w", from, err)
}
if localPath == "" {
localPath = resolvedPath
}
}
out = append(out, archivePromotion{
From: from,
To: to,
Required: required,
LocalPath: localPath,
Exists: exists,
})
}
return out, nil
}
func archiveSourceWorkDirs(env *Env, m *manifest.Manifest, runWorkDir string) []string {
candidates := make([]string, 0, 2)
if strings.TrimSpace(runWorkDir) != "" {
candidates = append(candidates, filepath.Clean(runWorkDir))
}
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" && m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
if sessionID != "" {
sessionWorkDir := filepath.Clean(artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID))
if sessionWorkDir != "" {
candidates = append(candidates, sessionWorkDir)
}
}
seen := make(map[string]struct{}, len(candidates))
out := make([]string, 0, len(candidates))
for _, c := range candidates {
if _, ok := seen[c]; ok {
continue
}
seen[c] = struct{}{}
out = append(out, c)
}
return out
}
func resolveWorkDirRelativePath(workDir, rel string) (string, error) {
rel = filepath.Clean(filepath.FromSlash(strings.TrimSpace(rel)))
if rel == "." || rel == "" {
return "", fmt.Errorf("relative path is required")
}
full := filepath.Join(workDir, rel)
cleanedWork := filepath.Clean(workDir)
cleanedFull := filepath.Clean(full)
relative, err := filepath.Rel(cleanedWork, cleanedFull)
if err != nil {
return "", fmt.Errorf("compute relative path: %w", err)
}
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path escapes workdir")
}
return cleanedFull, nil
}
func collectArchiveRunFiles(workDir, manifestPath string) ([]archiveUploadFile, error) {
files := make([]archiveUploadFile, 0, 64)
for _, dirName := range archiveRunUploadDirs {
fullDir := filepath.Join(workDir, dirName)
info, err := os.Stat(fullDir)
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, fmt.Errorf("stat %q: %w", fullDir, err)
}
if !info.IsDir() {
continue
}
if err := filepath.WalkDir(fullDir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(workDir, path)
if err != nil {
return fmt.Errorf("relative path from %q to %q: %w", workDir, path, err)
}
rel = filepath.ToSlash(rel)
files = append(files, archiveUploadFile{
RelativePath: rel,
LocalPath: path,
})
return nil
}); err != nil {
return nil, fmt.Errorf("walk %q: %w", fullDir, err)
}
}
manifestInfo, err := os.Stat(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("manifest.json not found (checked path %q)", manifestPath)
}
return nil, fmt.Errorf("stat %q: %w", manifestPath, err)
}
if manifestInfo.IsDir() {
return nil, fmt.Errorf("manifest path %q is a directory", manifestPath)
}
files = append(files, archiveUploadFile{
RelativePath: "manifest.json",
LocalPath: manifestPath,
})
sort.Slice(files, func(i, j int) bool {
return files[i].RelativePath < files[j].RelativePath
})
return files, nil
}
func resolveArchiveManifestSource(env *Env, m *manifest.Manifest, workDir string) (string, error) {
candidates := make([]string, 0, 3)
candidates = append(candidates, filepath.Join(workDir, "manifest.json"))
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if sessionID == "" && m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
if sessionID != "" {
sessionManifest := filepath.Join(artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID), "manifest.json")
candidates = append(candidates, sessionManifest)
}
seen := make(map[string]struct{}, len(candidates))
for _, candidate := range candidates {
clean := filepath.Clean(candidate)
if _, ok := seen[clean]; ok {
continue
}
seen[clean] = struct{}{}
info, err := os.Stat(clean)
if err != nil {
if os.IsNotExist(err) {
continue
}
return "", fmt.Errorf("stat %q: %w", clean, err)
}
if info.IsDir() {
continue
}
return clean, nil
}
return "", fmt.Errorf("manifest.json not found in run workdir or session workdir")
}
func writeCurrentManifestSnapshot(m *manifest.Manifest, archiveMetadata map[string]any) (string, error) {
if m == nil {
return "", fmt.Errorf("manifest is required")
}
clone := *m
clone.Stages = make(map[string]*manifest.StageRecord, len(m.Stages))
for name, sr := range m.Stages {
if sr == nil {
continue
}
stageCopy := *sr
if sr.Outputs != nil {
stageCopy.Outputs = append([]manifest.ArtifactRecord(nil), sr.Outputs...)
}
if sr.Logs != nil {
stageCopy.Logs = append([]string(nil), sr.Logs...)
}
if sr.GeneratedConfigs != nil {
stageCopy.GeneratedConfigs = append([]string(nil), sr.GeneratedConfigs...)
}
if sr.Metadata != nil {
metaCopy := make(map[string]any, len(sr.Metadata))
for k, v := range sr.Metadata {
metaCopy[k] = v
}
stageCopy.Metadata = metaCopy
}
clone.Stages[name] = &stageCopy
}
now := time.Now().UTC()
clone.MarkStageSucceeded("archive", now, nil)
if sr := clone.Stages["archive"]; sr != nil {
sr.Metadata = archiveMetadata
}
data, err := json.MarshalIndent(&clone, "", " ")
if err != nil {
return "", fmt.Errorf("marshal manifest: %w", err)
}
data = append(data, '\n')
tmp, err := os.CreateTemp("", "narratio-current-manifest-*.json")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("write temp manifest: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp manifest: %w", err)
}
return path, nil
}
func writeCurrentRunIDPointer(runID string) (string, error) {
tmp, err := os.CreateTemp("", "narratio-current-run-id-*.txt")
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if _, err := tmp.WriteString(runID + "\n"); err != nil {
_ = tmp.Close()
return "", fmt.Errorf("write temp run id pointer: %w", err)
}
if err := tmp.Close(); err != nil {
return "", fmt.Errorf("close temp run id pointer: %w", err)
}
return path, nil
}
func archiveMetadataPreview(
bucket, runPrefix, sessionPrefix string,
runUploaded []string,
promotedUploaded []string,
skippedOptional []string,
currentManifestKey string,
) map[string]any {
return map[string]any{
"stage": "archive",
"uploaded": true,
"s3_bucket": bucket,
"s3_run_prefix": runPrefix,
"run_files_uploaded": len(runUploaded),
"run_uploaded_paths": append([]string(nil), runUploaded...),
"promoted_files_uploaded": len(promotedUploaded),
"promoted_paths": append([]string(nil), promotedUploaded...),
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
"current_manifest_key": currentManifestKey,
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
"current_pointer_written": false,
"audio_upload_skipped": true,
}
}

View File

@@ -0,0 +1,346 @@
package stage
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestArchiveSkipsWhenDisabled(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.Enabled = boolPtr(false)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["skipped"] != true {
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads when archive disabled")
}
}
func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.UploadRun = boolPtr(false)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["skipped"] != true {
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads when upload_run disabled")
}
}
func TestArchiveFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
env, m, _ := archiveFixture(t)
m.Stages["trim"].Status = manifest.StatusFailed
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), `prerequisite stage "trim"`) {
t.Fatalf("Run() error = %v, want prerequisite failure", err)
}
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
t.Fatalf("unexpected uploads on prerequisite failure")
}
}
func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
runPrefix := m.S3RunPrefix
sessionPrefix := m.S3SessionPrefix
wantRunUploads := []string{
"artifacts/session_recap.md",
"config/audita.generated.yml",
"inputs/session.yml",
"logs/audita.stderr.log",
"manifest.json",
"reports/audita.report.json",
"transcripts/raw/speaker.json",
"transcripts/trimmed.json",
}
for _, rel := range wantRunUploads {
key := runPrefix + rel
if _, ok := fake.Objects[key]; !ok {
t.Fatalf("missing run upload key %q", key)
}
}
trimmedKey := sessionPrefix + "transcripts/trimmed.json"
recapKey := sessionPrefix + "artifacts/session_recap.md"
if _, ok := fake.Objects[trimmedKey]; !ok {
t.Fatalf("missing promoted key %q", trimmedKey)
}
if _, ok := fake.Objects[recapKey]; !ok {
t.Fatalf("missing promoted key %q", recapKey)
}
currentManifestKey := sessionPrefix + "current/manifest.json"
currentRunIDKey := sessionPrefix + "current/run_id.txt"
if _, ok := fake.Objects[currentManifestKey]; !ok {
t.Fatalf("missing current manifest key %q", currentManifestKey)
}
if _, ok := fake.Objects[currentRunIDKey]; !ok {
t.Fatalf("missing current run pointer key %q", currentRunIDKey)
}
if got := string(fake.Objects[currentRunIDKey].Data); got != m.RunID+"\n" {
t.Fatalf("run pointer contents = %q, want %q", got, m.RunID+"\\n")
}
audioKey := runPrefix + "audio/speaker.flac"
if _, ok := fake.Objects[audioKey]; ok {
t.Fatalf("audio key %q should not be uploaded", audioKey)
}
uploads := fake.Uploads
if len(uploads) == 0 {
t.Fatal("expected uploads")
}
if uploads[len(uploads)-1].Key != currentRunIDKey {
t.Fatalf("last upload key = %q, want current run pointer key %q", uploads[len(uploads)-1].Key, currentRunIDKey)
}
if result.Metadata["current_pointer_written"] != true {
t.Fatalf("metadata = %#v, want current_pointer_written=true", result.Metadata)
}
if result.Metadata["promoted_files_uploaded"] != 2 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
}
}
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
env, m, workDir := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "published/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "published/recap.md", Required: boolPtr(true)},
}
writeStageTestFile(t, filepath.Join(workDir, "published", "ignored.txt"), "ignore\n")
_, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
fake := env.ObjectStore.(*storage.FakeBackend)
if _, ok := fake.Objects[m.S3SessionPrefix+"published/trimmed.json"]; !ok {
t.Fatalf("missing custom promoted trimmed key")
}
if _, ok := fake.Objects[m.S3SessionPrefix+"published/recap.md"]; !ok {
t.Fatalf("missing custom promoted recap key")
}
}
func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/optional.md", To: "artifacts/optional.md", Required: boolPtr(false)},
}
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
got, _ := result.Metadata["skipped_optional_promotions"].([]string)
want := []string{"artifacts/optional.md"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("skipped_optional_promotions = %#v, want %#v", got, want)
}
}
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
env, m, _ := archiveFixture(t)
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
t.Fatalf("Run() error = %v, want required promotion missing failure", err)
}
}
func TestArchivePromotionFallsBackToSessionWorkDir(t *testing.T) {
env, m, runWorkDir := archiveFixture(t)
sessionWorkDir := artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, m.SessionID)
sessionTrimmed := filepath.Join(sessionWorkDir, "transcripts", "trimmed.json")
if err := os.Remove(filepath.Join(runWorkDir, "transcripts", "trimmed.json")); err != nil {
t.Fatalf("remove run scoped trimmed transcript: %v", err)
}
writeStageTestFile(t, sessionTrimmed, "{}\n")
result, err := archiveStage{}.Run(context.Background(), env, m)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if result.Metadata["promoted_files_uploaded"] != 2 {
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
}
fake := env.ObjectStore.(*storage.FakeBackend)
if _, ok := fake.Objects[m.S3SessionPrefix+"transcripts/trimmed.json"]; !ok {
t.Fatalf("missing promoted trimmed key from session fallback")
}
}
func TestArchiveDoesNotWriteCurrentPointerWhenPromotionUploadFails(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
origUploadErr := fake.UploadErr
fake.UploadErr = nil
failingKey := trimmedKey
fake.Uploads = nil
originalUpload := fake.Upload
_ = originalUpload
// Use UploadErr toggle by checking call sequence in postcondition.
// First failure point is promotion upload; simulate by setting error immediately before promotion key write.
// We cannot hook FakeBackend per-key without changing public behavior; use dedicated backend wrapper instead.
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: failingKey}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "promoted output") {
t.Fatalf("Run() error = %v, want promotion upload failure", err)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
t.Fatalf("unexpected current pointer write on promotion failure")
}
fake.UploadErr = origUploadErr
}
func TestArchiveDoesNotWriteCurrentPointerWhenCurrentManifestUploadFails(t *testing.T) {
env, m, _ := archiveFixture(t)
fake := env.ObjectStore.(*storage.FakeBackend)
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: m.S3SessionPrefix + "current/manifest.json"}
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "current manifest") {
t.Fatalf("Run() error = %v, want current manifest upload failure", err)
}
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
t.Fatalf("unexpected current pointer write when current manifest upload fails")
}
}
func TestArchiveFailsWithoutObjectStore(t *testing.T) {
env, m, _ := archiveFixture(t)
env.ObjectStore = nil
_, err := archiveStage{}.Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "object store") {
t.Fatalf("Run() error = %v, want object store backend failure", err)
}
}
func archiveFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
t.Helper()
root := t.TempDir()
runID := "20260516T010203Z-1a2b3c4d"
campaign := "forsaken"
sessionID := "2026-04-19"
workDir := filepath.Join(root, "work", campaign, sessionID, runID)
writeStageTestFile(t, filepath.Join(workDir, "inputs", "session.yml"), "session_id: 2026-04-19\n")
writeStageTestFile(t, filepath.Join(workDir, "transcripts", "raw", "speaker.json"), "{}\n")
writeStageTestFile(t, filepath.Join(workDir, "transcripts", "trimmed.json"), "{}\n")
writeStageTestFile(t, filepath.Join(workDir, "artifacts", "session_recap.md"), "# recap\n")
writeStageTestFile(t, filepath.Join(workDir, "reports", "audita.report.json"), "{}\n")
writeStageTestFile(t, filepath.Join(workDir, "config", "audita.generated.yml"), "key: value\n")
writeStageTestFile(t, filepath.Join(workDir, "logs", "audita.stderr.log"), "stderr\n")
writeStageTestFile(t, filepath.Join(workDir, "audio", "speaker.flac"), "flac")
writeStageTestFile(t, filepath.Join(workDir, "manifest.json"), "{}\n")
m := manifest.New(sessionID, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC))
m.Campaign = campaign
m.RunID = runID
m.LocalWorkDir = workDir
m.S3Bucket = "my-dnd-archive"
m.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", campaign, sessionID)
m.S3RunPrefix = artifacts.S3RunPrefix(m.S3SessionPrefix, runID)
for _, name := range archivePrerequisiteStages {
m.MarkStageSucceeded(name, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), nil)
}
env := &Env{
Config: &config.Config{
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{
S3: &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
},
},
Archive: &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
PromoteArtifacts: []config.ArchivePromotionRule{
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
},
},
},
Session: &config.SessionConfig{
SessionID: sessionID,
Campaign: campaign,
},
},
ObjectStore: &storage.FakeBackend{},
}
return env, m, workDir
}
type promotionFailingStore struct {
delegate *storage.FakeBackend
failKey string
}
func (s *promotionFailingStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
return s.delegate.List(ctx, prefix)
}
func (s *promotionFailingStore) Download(ctx context.Context, key, localPath string) error {
return s.delegate.Download(ctx, key, localPath)
}
func (s *promotionFailingStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
return storage.ObjectInfo{}, errors.New("forced upload failure")
}
return s.delegate.Upload(ctx, localPath, key, opts)
}
func (s *promotionFailingStore) Exists(ctx context.Context, key string) (bool, error) {
return s.delegate.Exists(ctx, key)
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -7,6 +7,7 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
@@ -86,10 +87,15 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
stderrPath := filepath.Join(paths.LogsDir, "seriatim.stderr.log")
genCfgPath := filepath.Join(paths.ConfigDir, "seriatim.generated.yml")
normalizedInputs, normalizeLogs, normalizeConfigs, normalizeMeta, err := normalizeMergeInputs(ctx, env, inputs, paths)
if err != nil {
return nil, err
}
reportEnabled := env.Config.Pipeline.Seriatim.Report != nil && *env.Config.Pipeline.Seriatim.Report
req := seriatim.MergeRequest{
GeneratedConfigPath: genCfgPath,
InputTranscriptPaths: inputs,
InputTranscriptPaths: normalizedInputs,
OutputMergedTranscriptPath: mergedPath,
ReportPath: "",
SpeakersPath: speakersPath,
@@ -146,8 +152,11 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
meta := map[string]any{
"stage": "merge",
"input_transcripts_count": len(inputs),
"input_transcripts_count": len(normalizedInputs),
"input_transcript_paths": inputs,
"normalized_inputs_count": len(normalizedInputs),
"normalized_input_paths": normalizedInputs,
"normalize_inputs": normalizeMeta,
"output_schema": env.Config.Pipeline.Seriatim.OutputSchema,
"coalesce_gap": coalesceGap,
"report_enabled": reportEnabled,
@@ -172,12 +181,97 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
return &StageResult{
Outputs: outputs,
Logs: []string{stdoutPath, stderrPath},
GeneratedConfigs: []string{genCfgPath},
Logs: append(normalizeLogs, stdoutPath, stderrPath),
GeneratedConfigs: append(normalizeConfigs, genCfgPath),
Metadata: meta,
}, nil
}
type normalizeMergeInputMeta struct {
InputPath string `json:"input_path"`
OutputPath string `json:"output_path"`
StdoutLogPath string `json:"stdout_log_path"`
StderrLogPath string `json:"stderr_log_path"`
GeneratedConfig string `json:"generated_config_path"`
DurationMs int64 `json:"duration_ms"`
ExitCode int `json:"exit_code"`
InvokedBinary string `json:"invoked_binary"`
OutputSchema string `json:"output_schema"`
AdapterReportPath string `json:"adapter_report_path,omitempty"`
AdapterOutputPath string `json:"adapter_output_path,omitempty"`
}
func normalizeMergeInputs(ctx context.Context, env *Env, rawInputs []string, paths artifacts.SessionPaths) ([]string, []string, []string, []normalizeMergeInputMeta, error) {
normalizedInputs := make([]string, 0, len(rawInputs))
logs := make([]string, 0, len(rawInputs)*2)
configs := make([]string, 0, len(rawInputs))
meta := make([]normalizeMergeInputMeta, 0, len(rawInputs))
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
if err := os.MkdirAll(normalizedDir, 0o755); err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: ensure normalized transcripts directory %q: %w", normalizedDir, err)
}
var timeout time.Duration
timeoutRaw := strings.TrimSpace(env.Config.Pipeline.Seriatim.Timeout)
if timeoutRaw != "" {
parsed, err := time.ParseDuration(timeoutRaw)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: parse seriatim timeout %q: %w", env.Config.Pipeline.Seriatim.Timeout, err)
}
timeout = parsed
}
for _, input := range rawInputs {
base := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input))
outPath := filepath.Join(normalizedDir, base+".normalized.json")
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stdout.log")
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stderr.log")
cfgPath := filepath.Join(paths.ConfigDir, "seriatim.normalize."+base+".generated.yml")
req := seriatim.NormalizeRequest{
Binary: env.Config.Pipeline.Seriatim.Binary,
InputTranscriptPath: input,
OutputNormalizedPath: outPath,
OutputSchema: env.Config.Pipeline.Seriatim.OutputSchema,
ReportPath: "",
StdoutLogPath: stdoutPath,
StderrLogPath: stderrPath,
GeneratedConfigPath: cfgPath,
Timeout: timeout,
}
res, err := env.Seriatim.Normalize(ctx, req)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: normalize input %q failed: %w", input, err)
}
finalOutputPath := outPath
if strings.TrimSpace(res.OutputNormalizedPath) != "" {
finalOutputPath = strings.TrimSpace(res.OutputNormalizedPath)
}
if err := validateTranscriptJSONFile(finalOutputPath); err != nil {
return nil, nil, nil, nil, fmt.Errorf("merge: normalized transcript %q invalid (input %q): %w", finalOutputPath, input, err)
}
normalizedInputs = append(normalizedInputs, finalOutputPath)
logs = append(logs, stdoutPath, stderrPath)
configs = append(configs, cfgPath)
meta = append(meta, normalizeMergeInputMeta{
InputPath: input,
OutputPath: finalOutputPath,
StdoutLogPath: stdoutPath,
StderrLogPath: stderrPath,
GeneratedConfig: cfgPath,
DurationMs: res.Duration.Milliseconds(),
ExitCode: res.ExitCode,
InvokedBinary: res.InvokedBinary,
OutputSchema: res.OutputSchema,
AdapterReportPath: res.ReportPath,
AdapterOutputPath: res.OutputNormalizedPath,
})
}
return normalizedInputs, logs, configs, meta, nil
}
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
fromManifest := make([]string, 0)
if m != nil && m.Stages != nil {

View File

@@ -2,6 +2,7 @@ package stage
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
@@ -14,6 +15,22 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type normalizeDirAssertingRunner struct {
*seriatim.FakeRunner
ExpectedDir string
}
func (r *normalizeDirAssertingRunner) Normalize(ctx context.Context, req seriatim.NormalizeRequest) (seriatim.NormalizeResult, error) {
info, err := os.Stat(r.ExpectedDir)
if err != nil {
return seriatim.NormalizeResult{}, fmt.Errorf("normalized directory check failed for %q: %w", r.ExpectedDir, err)
}
if !info.IsDir() {
return seriatim.NormalizeResult{}, fmt.Errorf("normalized path %q exists but is not a directory", r.ExpectedDir)
}
return r.FakeRunner.Normalize(ctx, req)
}
func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
@@ -54,6 +71,17 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if len(req.InputTranscriptPaths) != 2 {
t.Fatalf("input transcripts = %#v, want 2", req.InputTranscriptPaths)
}
if len(fake.NormalizeRequests) != 2 {
t.Fatalf("normalize requests = %#v, want 2", fake.NormalizeRequests)
}
if fake.NormalizeRequests[0].InputTranscriptPath != inA || fake.NormalizeRequests[1].InputTranscriptPath != inB {
t.Fatalf("normalize request inputs = %#v", fake.NormalizeRequests)
}
for _, mergeIn := range req.InputTranscriptPaths {
if !strings.Contains(mergeIn, filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("merge input path = %q, want normalized input path", mergeIn)
}
}
if len(result.Outputs) != 2 {
t.Fatalf("outputs len = %d, want 2", len(result.Outputs))
@@ -64,11 +92,11 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if result.Outputs[1].Kind != "seriatim_report" {
t.Fatalf("output[1] kind = %q, want seriatim_report", result.Outputs[1].Kind)
}
if len(result.Logs) != 2 {
t.Fatalf("logs = %#v, want 2 paths", result.Logs)
if len(result.Logs) != 6 {
t.Fatalf("logs = %#v, want 6 paths (4 normalize + 2 merge)", result.Logs)
}
if len(result.GeneratedConfigs) != 1 {
t.Fatalf("generated configs = %#v, want 1 path", result.GeneratedConfigs)
if len(result.GeneratedConfigs) != 3 {
t.Fatalf("generated configs = %#v, want 3 paths (2 normalize + 1 merge)", result.GeneratedConfigs)
}
meta := result.Metadata
@@ -84,6 +112,12 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
if meta["input_transcripts_count"] != 2 {
t.Fatalf("metadata input_transcripts_count = %#v, want 2", meta["input_transcripts_count"])
}
if meta["normalized_inputs_count"] != 2 {
t.Fatalf("metadata normalized_inputs_count = %#v, want 2", meta["normalized_inputs_count"])
}
if _, ok := meta["normalize_inputs"]; !ok {
t.Fatalf("metadata normalize_inputs missing: %#v", meta)
}
}
func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
@@ -152,6 +186,9 @@ func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testin
if len(fake.Requests) != 1 || len(fake.Requests[0].InputTranscriptPaths) != 1 {
t.Fatalf("fallback inputs = %#v", fake.Requests)
}
if len(fake.NormalizeRequests) != 1 {
t.Fatalf("normalize requests = %#v, want 1", fake.NormalizeRequests)
}
}
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
@@ -180,8 +217,75 @@ func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t
if len(got) != 1 {
t.Fatalf("input transcript paths = %#v, want len 1", got)
}
if got[0] != filepath.Clean(rawPath) {
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
}
}
func TestMergeStageCreatesNormalizedRawDirectoryBeforeNormalize(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, rawPath, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
if err := os.RemoveAll(normalizedDir); err != nil {
t.Fatalf("remove normalized dir: %v", err)
}
env.Seriatim = &normalizeDirAssertingRunner{
FakeRunner: &seriatim.FakeRunner{},
ExpectedDir: normalizedDir,
}
if _, err := (mergeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("merge.Run() error = %v", err)
}
}
func TestMergeStageFailsWhenNormalizeAdapterFails(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, in, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
env.Seriatim = &seriatim.FakeRunner{NormalizeErr: context.DeadlineExceeded}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "normalize input") {
t.Fatalf("error = %q", err.Error())
}
}
func TestMergeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
env, m := setupMergeEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, in, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
badNormalized := filepath.Join(paths.ArtifactsDir, "bad.normalized.json")
writeFile(t, badNormalized, "not-json")
env.Seriatim = &seriatim.FakeRunner{
NormalizeResult: seriatim.NormalizeResult{
OutputNormalizedPath: badNormalized,
},
}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "normalized transcript") {
t.Fatalf("error = %q", err.Error())
}
}
@@ -211,8 +315,8 @@ func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
if len(got) != 1 {
t.Fatalf("input transcript paths = %#v, want len 1", got)
}
if got[0] != filepath.Clean(rawPath) {
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
}
}

View File

@@ -3,10 +3,8 @@ package stage
import (
"context"
"fmt"
"path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -47,25 +45,7 @@ func (s placeholderStage) Run(ctx context.Context, env *Env, m *manifest.Manifes
return result, nil
}
paths := env.ArtifactStore.SessionPaths(sessionID)
switch s.name {
case "archive":
if env.Storage != nil {
req := storage.ArchiveRequest{
SessionID: sessionID,
ManifestPath: paths.ManifestPath,
Items: []storage.ArchiveItem{{
Kind: "artifact",
LocalPath: filepath.Join(paths.ArtifactsDir, "session-log.md"),
RemoteKey: "sessions/" + sessionID + "/artifacts/session-log.md",
}},
}
_, err := env.Storage.Archive(ctx, req)
if err != nil {
return nil, fmt.Errorf("placeholder archive adapter call failed: %w", err)
}
}
case "notify":
if env.Notifier != nil {
req := notify.SendRequest{
@@ -95,7 +75,7 @@ func All() []Stage {
normalizeStage{},
trimStage{},
analyzeStage{},
placeholderStage{name: "archive"},
archiveStage{},
placeholderStage{name: "notify"},
}
}

View File

@@ -49,7 +49,19 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
Config: &config.Config{
SessionPath: sessionPath,
PipelinePath: pipelinePath,
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: root}},
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: root},
Storage: config.StorageConfig{
S3: &config.StorageS3Config{
Bucket: "my-dnd-archive",
RootPrefix: "dnd",
},
},
Archive: &config.ArchiveConfig{
Enabled: boolPtr(true),
UploadRun: boolPtr(true),
},
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Inputs: config.SessionInputsConfig{
@@ -66,10 +78,24 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
Audita: af,
Scriptorium: sc,
Storage: st,
ObjectStore: st,
Notifier: nf,
}
m := manifest.New("2026-05-03", time.Now().UTC())
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
m.RunID = "20260516T000000Z-abcdef12"
m.Campaign = "sample-campaign"
m.LocalWorkDir = filepath.Join(root, "work", "sample-campaign", "2026-05-03", m.RunID)
m.S3RunPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/" + m.RunID + "/"
m.S3Bucket = "my-dnd-archive"
if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "inputs"), 0o755); err != nil {
t.Fatalf("mkdir workdir inputs: %v", err)
}
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "inputs", "session.yml"), "session_id: 2026-05-03\n")
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "manifest.json"), "{}\n")
for _, s := range stages {
result, err := s.Run(context.Background(), env, m)
if err != nil {
@@ -150,6 +176,15 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
}
continue
}
if s.Name() == "archive" {
if result.Metadata["stage"] != "archive" {
t.Fatalf("archive metadata = %#v, want stage=archive", result.Metadata)
}
if result.Metadata["uploaded"] != true {
t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
}
continue
}
if result.Metadata["placeholder"] != true {
t.Fatalf("stage %q missing placeholder metadata", s.Name())
}
@@ -167,8 +202,11 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
if len(sc.RunRequests) != 0 {
t.Fatalf("scriptorium run calls = %d, want 0 when scriptorium config is absent", len(sc.RunRequests))
}
if len(st.Requests) != 1 {
t.Fatalf("storage calls = %d, want 1", len(st.Requests))
if len(st.Requests) != 0 {
t.Fatalf("storage archive calls = %d, want 0", len(st.Requests))
}
if _, ok := st.Objects["dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/"+m.RunID+"/manifest.json"]; !ok {
t.Fatalf("archive upload missing manifest key in fake object store")
}
if len(nf.Requests) != 1 {
t.Fatalf("notify calls = %d, want 1", len(nf.Requests))
@@ -181,7 +219,6 @@ func TestPlaceholderAdapterErrorPropagation(t *testing.T) {
env *Env
wantErr string
}{
{stageName: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("sterr")}}, wantErr: "archive"},
{stageName: "notify", env: &Env{Notifier: &notify.FakeSender{Err: errors.New("nerr")}}, wantErr: "notify"},
}

View File

@@ -90,6 +90,12 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
Modules: append([]string(nil), env.Config.Pipeline.Audita.Modules...),
BaseURL: env.Config.Pipeline.Audita.BaseURL,
Model: env.Config.Pipeline.Audita.Model,
TranscriptDescription: env.Config.Pipeline.Audita.TranscriptDescription,
ConfigPath: env.Config.Pipeline.Audita.ConfigPath,
OutputSchema: env.Config.Pipeline.Audita.OutputSchema,
WorkDirRetention: env.Config.Pipeline.Audita.WorkDirRetention,
TotalLLMConcurrency: env.Config.Pipeline.Audita.TotalLLMConcurrency,
ProposalLLMConcurrency: env.Config.Pipeline.Audita.ProposalLLMConcurrency,
ValidationModel: env.Config.Pipeline.Audita.ValidationModel,
ValidationLLMConcurrency: env.Config.Pipeline.Audita.ValidationLLMConcurrency,
StdoutLogPath: stdoutPath,
@@ -141,44 +147,52 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
if env.Config.Pipeline.Audita.ValidationLLMConcurrency != nil {
validationConcurrency = *env.Config.Pipeline.Audita.ValidationLLMConcurrency
}
var llmConcurrency any
if env.Config.Pipeline.Audita.LLMConcurrency != nil {
llmConcurrency = *env.Config.Pipeline.Audita.LLMConcurrency
var totalLLMConcurrency any
if env.Config.Pipeline.Audita.TotalLLMConcurrency != nil {
totalLLMConcurrency = *env.Config.Pipeline.Audita.TotalLLMConcurrency
}
var proposalLLMConcurrency any
if env.Config.Pipeline.Audita.ProposalLLMConcurrency != nil {
proposalLLMConcurrency = *env.Config.Pipeline.Audita.ProposalLLMConcurrency
}
meta := map[string]any{
"stage": "polish",
"merged_transcript_path": mergedPath,
"merged_transcript_source": source,
"glossary_path": glossaryPath,
"output_path": finalProcessedPath,
"report_path": finalReportPath,
"audita_work_dir": workDir,
"report_enabled": reportEnabled,
"modules": append([]string(nil), req.Modules...),
"base_url": req.BaseURL,
"model": req.Model,
"validation_model": req.ValidationModel,
"llm_concurrency": llmConcurrency,
"validation_llm_concurrency": validationConcurrency,
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"timeout": env.Config.Pipeline.Audita.Timeout,
"binary": env.Config.Pipeline.Audita.Binary,
"generated_config_path": generatedConfigPath,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_processed_output_path": res.ProcessedTranscriptPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config_path": res.GeneratedConfigPath,
"adapter_work_dir": res.WorkDir,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"credential_present": false,
"primary_llm_concurrency_via_env": false,
"stage": "polish",
"merged_transcript_path": mergedPath,
"merged_transcript_source": source,
"glossary_path": glossaryPath,
"output_path": finalProcessedPath,
"report_path": finalReportPath,
"audita_work_dir": workDir,
"report_enabled": reportEnabled,
"modules": append([]string(nil), req.Modules...),
"base_url": req.BaseURL,
"model": req.Model,
"transcript_description": req.TranscriptDescription,
"config_path": req.ConfigPath,
"output_schema": req.OutputSchema,
"work_dir_retention": req.WorkDirRetention,
"validation_model": req.ValidationModel,
"total_llm_concurrency": totalLLMConcurrency,
"proposal_llm_concurrency": proposalLLMConcurrency,
"validation_llm_concurrency": validationConcurrency,
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"timeout": env.Config.Pipeline.Audita.Timeout,
"binary": env.Config.Pipeline.Audita.Binary,
"generated_config_path": generatedConfigPath,
"stdout_log_path": stdoutPath,
"stderr_log_path": stderrPath,
"adapter_duration_ms": res.Duration.Milliseconds(),
"adapter_exit_code": res.ExitCode,
"adapter_invoked_binary": res.InvokedBinary,
"adapter_processed_output_path": res.ProcessedTranscriptPath,
"adapter_report_path": res.ReportPath,
"adapter_generated_config_path": res.GeneratedConfigPath,
"adapter_work_dir": res.WorkDir,
"adapter_stdout_log_path": res.StdoutLogPath,
"adapter_stderr_log_path": res.StderrLogPath,
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
"credential_present": false,
}
if res.Metadata != nil {
meta["adapter_metadata"] = res.Metadata
@@ -188,9 +202,6 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
if value, ok := res.Metadata["credential_env_var"]; ok {
meta["credential_env_var"] = value
}
if value, ok := res.Metadata["primary_llm_concurrency_via_env"]; ok {
meta["primary_llm_concurrency_via_env"] = value
}
}
return &StageResult{

View File

@@ -60,6 +60,24 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
if req.ValidationModel != "openrouter/google/gemma-4-31b-it" {
t.Fatalf("validation model = %q", req.ValidationModel)
}
if req.TranscriptDescription != "Campaign Session 42" {
t.Fatalf("transcript description = %q", req.TranscriptDescription)
}
if req.ConfigPath != "/etc/audita/config.yml" {
t.Fatalf("config path = %q", req.ConfigPath)
}
if req.OutputSchema != "audita-v1" {
t.Fatalf("output schema = %q", req.OutputSchema)
}
if req.WorkDirRetention != "auto" {
t.Fatalf("work dir retention = %q", req.WorkDirRetention)
}
if req.TotalLLMConcurrency == nil || *req.TotalLLMConcurrency != 3 {
t.Fatalf("total llm concurrency = %#v, want 3", req.TotalLLMConcurrency)
}
if req.ProposalLLMConcurrency == nil || *req.ProposalLLMConcurrency != 2 {
t.Fatalf("proposal llm concurrency = %#v, want 2", req.ProposalLLMConcurrency)
}
if req.ValidationLLMConcurrency == nil || *req.ValidationLLMConcurrency != 2 {
t.Fatalf("validation llm concurrency = %#v, want 2", req.ValidationLLMConcurrency)
}
@@ -89,6 +107,15 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
if result.Metadata["audita_work_dir"] != filepath.Join(paths.ArtifactsDir, "audita-work") {
t.Fatalf("metadata audita_work_dir = %#v", result.Metadata["audita_work_dir"])
}
if result.Metadata["total_llm_concurrency"] != 3 {
t.Fatalf("metadata total_llm_concurrency = %#v, want 3", result.Metadata["total_llm_concurrency"])
}
if result.Metadata["proposal_llm_concurrency"] != 2 {
t.Fatalf("metadata proposal_llm_concurrency = %#v, want 2", result.Metadata["proposal_llm_concurrency"])
}
if result.Metadata["output_schema"] != "audita-v1" {
t.Fatalf("metadata output_schema = %#v, want audita-v1", result.Metadata["output_schema"])
}
}
func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
@@ -221,7 +248,8 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
report := true
llmConcurrency := 1
totalLLMConcurrency := 3
proposalLLMConcurrency := 2
validationLLMConcurrency := 2
cfg := &config.Config{
@@ -236,8 +264,13 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
Modules: []string{"glossary", "homophones", "grammar"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
TranscriptDescription: "Campaign Session 42",
ConfigPath: "/etc/audita/config.yml",
OutputSchema: "audita-v1",
WorkDirRetention: "auto",
ValidationModel: "openrouter/google/gemma-4-31b-it",
LLMConcurrency: &llmConcurrency,
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
ValidationLLMConcurrency: &validationLLMConcurrency,
Report: &report,
},

View File

@@ -6,10 +6,12 @@ import (
"encoding/hex"
"fmt"
"os"
"path"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -33,7 +35,7 @@ func (prepareStage) Declares() IODecl {
}
}
func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("prepare: stage environment config is required")
}
@@ -89,13 +91,12 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
}
}
resolvedAudio, err := resolveAudioFiles(sessionDir, env.Config.Session.Inputs)
resolvedLocalAudio, useS3Audio, err := resolveAudioInputs(sessionDir, env.Config.Session.Inputs)
if err != nil {
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
}
copiedByDest := map[string]string{}
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedAudio))
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
registerInput := func(kind, path, checksum string) {
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
}
@@ -134,19 +135,14 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
registerInput(cfgFile.kind, cfgFile.dst, checksum)
}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return nil, fmt.Errorf("prepare: duplicate audio basename %q from %q and %q", base, prev, src)
if useS3Audio {
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return nil, fmt.Errorf("prepare: materialize audio %q: %w", base, err)
} else {
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
return nil, fmt.Errorf("prepare: %w", err)
}
registerInput("audio", dst, checksum)
}
sort.Slice(inputs, func(i, j int) bool {
@@ -162,7 +158,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
"prepared": true,
"stage": "prepare",
"inputs_count": len(inputs),
"audio_files_resolved": len(resolvedAudio),
"audio_files_resolved": countAudioInputs(inputs),
},
}, nil
}
@@ -171,7 +167,23 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
return yaml.Marshal(cfg)
}
func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
if inputs.AudioS3 != nil {
if hasLocal {
return nil, false, fmt.Errorf("audio_dir/audio_files and audio_s3 are mutually exclusive")
}
return nil, true, nil
}
local, err := resolveLocalAudioFiles(sessionDir, inputs)
if err != nil {
return nil, false, err
}
return local, false, nil
}
func resolveLocalAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
if len(inputs.AudioFiles) > 0 {
out := make([]string, 0, len(inputs.AudioFiles))
for _, p := range inputs.AudioFiles {
@@ -223,6 +235,142 @@ func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]
return out, nil
}
func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolvedAudio []string, registerInput func(kind, path, checksum string)) error {
copiedByDest := map[string]string{}
for _, src := range resolvedAudio {
base := filepath.Base(src)
if prev, exists := copiedByDest[base]; exists && prev != src {
return fmt.Errorf("duplicate audio basename %q from %q and %q", base, prev, src)
}
copiedByDest[base] = src
dst := filepath.Join(paths.AudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
if err != nil {
return fmt.Errorf("materialize audio %q: %w", base, err)
}
registerInput("audio", dst, checksum)
}
return nil
}
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
if env.ObjectStore == nil {
return fmt.Errorf("s3 audio input requires object store backend")
}
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
}
campaign := strings.TrimSpace(env.Config.Session.Campaign)
if campaign == "" {
return fmt.Errorf("session campaign is required for s3 audio input")
}
runID := strings.TrimSpace(m.RunID)
if runID == "" {
return fmt.Errorf("run id is required for s3 audio input")
}
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
objects, err := env.ObjectStore.List(ctx, audioPrefix)
if err != nil {
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
}
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
for _, obj := range objects {
key := strings.TrimSpace(obj.Key)
if key == "" || strings.HasSuffix(key, "/") {
continue
}
if !isFlac(key) {
continue
}
audioObjects = append(audioObjects, obj)
}
sort.Slice(audioObjects, func(i, j int) bool {
return audioObjects[i].Key < audioObjects[j].Key
})
if len(audioObjects) == 0 {
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
}
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolAudioDir == "" {
spoolAudioDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, campaign, sessionID, runID)
}
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
}
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
}
seenBase := map[string]string{}
for _, obj := range audioObjects {
base := path.Base(obj.Key)
if prev, exists := seenBase[base]; exists && prev != obj.Key {
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
}
seenBase[base] = obj.Key
spoolPath := filepath.Join(spoolAudioDir, base)
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
}
workPath := filepath.Join(workAudioDir, base)
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
if err != nil {
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
}
*inputs = append(*inputs, manifest.InputRecord{
Kind: "audio",
Path: workPath,
Checksum: checksum,
Source: "s3",
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
S3Key: obj.Key,
S3Size: obj.Size,
S3ETag: obj.ETag,
SpoolPath: spoolPath,
})
}
return nil
}
func countAudioInputs(inputs []manifest.InputRecord) int {
count := 0
for _, in := range inputs {
if in.Kind == "audio" {
count++
}
}
return count
}
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return ""
}
if m != nil && strings.TrimSpace(m.LocalWorkDir) != "" {
return strings.TrimSpace(m.LocalWorkDir)
}
runID := ""
if m != nil {
runID = strings.TrimSpace(m.RunID)
}
if runID != "" {
return artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, env.Config.Session.Campaign, sessionID, runID)
}
return artifacts.SessionWorkDir(env.Config.Pipeline.Workspace.Root, sessionID)
}
func resolvePath(baseDir, p string) (string, error) {
trimmed := strings.TrimSpace(p)
if trimmed == "" {

View File

@@ -8,6 +8,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
@@ -150,6 +151,139 @@ func TestPrepareStageIdempotent(t *testing.T) {
}
}
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/bob.FLAC", Data: []byte("bob")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/ignore.txt", Data: []byte("x")})
env.ObjectStore = fake
result, err := (prepareStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if result == nil || result.Metadata["audio_files_resolved"] != 2 {
t.Fatalf("result metadata = %#v, want audio_files_resolved=2", result)
}
aliceWork := filepath.Join(m.LocalWorkDir, "audio", "alice.flac")
bobWork := filepath.Join(m.LocalWorkDir, "audio", "bob.FLAC")
aliceSpool := filepath.Join(m.LocalSpoolDir, "alice.flac")
bobSpool := filepath.Join(m.LocalSpoolDir, "bob.FLAC")
for _, p := range []string{aliceWork, bobWork, aliceSpool, bobSpool} {
if _, err := os.Stat(p); err != nil {
t.Fatalf("expected file %q: %v", p, err)
}
}
audioInputs := 0
for _, in := range m.Inputs {
if in.Kind != "audio" {
continue
}
audioInputs++
if in.Source != "s3" {
t.Fatalf("audio input source = %q, want s3", in.Source)
}
if in.S3Bucket != "my-dnd-archive" {
t.Fatalf("audio input bucket = %q", in.S3Bucket)
}
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
t.Fatalf("audio input missing provenance: %#v", in)
}
}
if audioInputs != 2 {
t.Fatalf("audio input count = %d, want 2", audioInputs)
}
}
func TestPrepareStageS3AudioFailures(t *testing.T) {
tests := []struct {
name string
setup func(env *Env, m *manifest.Manifest, fake *storage.FakeBackend)
wantError string
}{
{
name: "no flac files",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/readme.txt", Data: []byte("x")})
},
wantError: "no .flac files found",
},
{
name: "list error",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.ListErr = os.ErrPermission
},
wantError: "list s3 audio objects",
},
{
name: "download error",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
fake.DownloadErr = os.ErrPermission
},
wantError: "download s3 audio object",
},
{
name: "basename collision",
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/a/alice.flac", Data: []byte("a")})
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/b/alice.flac", Data: []byte("b")})
},
wantError: "duplicate s3 audio basename",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env, m := setupPrepareEnv(t)
env.Config.Session.Campaign = "forsaken"
env.Config.Session.Inputs.AudioDir = ""
env.Config.Session.Inputs.AudioFiles = nil
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
m.RunID = "20260515T031522Z-a1b2c3d4"
m.LocalWorkDir = artifacts.SessionRunWorkDir(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
fake := &storage.FakeBackend{}
tt.setup(env, m, fake)
env.ObjectStore = fake
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), tt.wantError) {
t.Fatalf("error = %v, want %q", err, tt.wantError)
}
})
}
}
func TestPrepareStageAudioSourceConflictFails(t *testing.T) {
env, m := setupPrepareEnv(t)
root := filepath.Dir(env.Config.SessionPath)
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
env.Config.Session.Inputs.AudioDir = "./audio"
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
_, err := (prepareStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("error = %v, want mutually exclusive error", err)
}
}
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
t.Helper()
workspace := t.TempDir()
@@ -170,6 +304,7 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
PipelinePath: pipelinePath,
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
Inputs: config.SessionInputsConfig{
AudioDir: "./audio",
SpeakersFile: "./speakers.yml",

View File

@@ -29,6 +29,7 @@ type Env struct {
Scriptorium scriptorium.Runner
Analyzer analyzer.Runner
Storage storage.Backend
ObjectStore storage.ObjectStore
Notifier notify.Sender
}