Compare commits
161 Commits
v0.8.2
...
13de820931
| Author | SHA1 | Date | |
|---|---|---|---|
| 13de820931 | |||
| 14ef59aaed | |||
| f387222fce | |||
| 9cb9008dfc | |||
| 083decc5b4 | |||
| 57cac5d3f7 | |||
| 0a772e03b4 | |||
| 0920062a38 | |||
| 39afe644eb | |||
| e3ee3de10a | |||
| 9c72db56e9 | |||
| bb2d606dbb | |||
| 9850767a8a | |||
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c | |||
| df58595d1e | |||
| c3c14e7468 | |||
| e7319ea016 | |||
| bd2d5e2496 | |||
| 115a44f629 | |||
| 18411dc5b5 | |||
| e23dc1ab6e | |||
| e1359ea227 | |||
| 7fdd99ec27 | |||
| a90231ce0c | |||
| ed879b8bb0 | |||
| 717451512a | |||
| 3ddb3a947b | |||
| c6632d5576 | |||
| ffc07922c7 | |||
| f3310d4d16 | |||
| 88cee96d8d | |||
| 2fece10215 | |||
| 0658f2f642 | |||
| a51228c803 | |||
| 4491fb5ccd | |||
| 30b905765c | |||
| 03eac70881 | |||
| 0f7e6b979f | |||
| c366912586 | |||
| 9fe44cd00d | |||
| 094b0d2532 | |||
| 98649f4d81 | |||
| 8a559efd5b | |||
| 72deccb4e2 | |||
| 5620fc5bcf | |||
| be57e675e0 | |||
| 3971443831 | |||
| a6b0c33e9f | |||
| 96b886e711 | |||
| 7d584ee6cd | |||
| 572a112c31 | |||
| ea87c335d6 | |||
| 7169ff04df | |||
| ef1f650bc0 | |||
| 0d02cb9fa0 | |||
| 0299b128cf | |||
| d723384888 | |||
| 54228055c8 | |||
| 23ed716450 | |||
| ab59bab044 | |||
| 71395bb076 | |||
| 79737edf79 | |||
| df2c765b7f | |||
| f050b9dd54 | |||
| 9c9cb54339 | |||
| 7657ec3ad6 | |||
| cee52aa092 | |||
| e920f3a8d5 | |||
| 591c529a09 | |||
| 7324c5a686 | |||
| d0936fb022 | |||
| 2aa074c5cf | |||
| 782d0cf3b9 | |||
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 | |||
| bc2ade38d9 | |||
| 5be831eb13 | |||
| cae4d99a89 | |||
| e09dc0512d | |||
| ae82bc1ce0 | |||
| 01eb7aa1aa | |||
| 2ca700195c | |||
| 2b08c34539 | |||
| 79f1fc1e09 | |||
| 9c753270bd | |||
| b907cb01aa | |||
| 7824afd4a5 | |||
| 2a4e1e912c | |||
| dd03c09d75 | |||
| 5bc8e8683f | |||
| 648001a8fe | |||
| 6684774f52 | |||
| f3b63bd5e5 | |||
| 23d6470b0f | |||
| 128449040f | |||
| 02ab106ade | |||
| c128970f58 | |||
| d001baa660 | |||
| c5c35cd3b4 | |||
| 574b1cde6c | |||
| ebb21b9201 | |||
| 958f446387 | |||
| 86caf4b222 | |||
| e38ed8ba97 | |||
| 3e79cf4724 | |||
| 859ae1ae10 | |||
| c63ecbab32 | |||
| 8480b74283 | |||
| 087869f7fa | |||
| 2b2a314d65 | |||
| 08b0f4edc5 | |||
| 571a289296 | |||
| 9f80635b42 | |||
| 11a3e174b6 | |||
| 9c5e5d6dc1 | |||
| c4e87f58c7 | |||
| 37daab7857 | |||
| 2356688cb9 | |||
| 1054b64d9f | |||
| 01fb02426c | |||
| 7dc79e052f | |||
| cb525c0f72 | |||
| 622677d038 | |||
| 550288e008 | |||
| e58e545686 | |||
| 6ff54c5a0f | |||
| 924b5d15c6 | |||
| b065663180 | |||
| 3ba564b00f | |||
| a3986cf0d6 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -2,6 +2,8 @@
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
.DS_Store
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
|
||||
481
README.md
481
README.md
@@ -1,459 +1,42 @@
|
||||
# narratio
|
||||
|
||||
`narratio` is a Go orchestration application for processing D&D session audio into transcripts and generated artifacts.
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts, validated Notarius extraction lanes, and generated
|
||||
artifacts.
|
||||
|
||||
## Current Implementation
|
||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||
publish, and restore support.
|
||||
|
||||
Implemented now:
|
||||
|
||||
- strict config loading/validation (`pipeline.yml` and `session.yml`)
|
||||
- local workspace/session layout, locking, and manifest persistence
|
||||
- resumable stage control (`run`, `plan`, `resume`, `run-stage`, `status`)
|
||||
- real `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` stages
|
||||
- real WhisperX, Seriatim, and Audita adapters
|
||||
- real Scriptorium subprocess adapter
|
||||
- optional Scriptorium render diagnostics (`render_debug`)
|
||||
|
||||
Not implemented yet:
|
||||
|
||||
- `notify` stage behavior
|
||||
- additional analyze artifacts beyond `session_recap`
|
||||
- generic DAG orchestration
|
||||
|
||||
## Config Files
|
||||
|
||||
Narratio expects two YAML files:
|
||||
|
||||
- `pipeline.yml`: pipeline/workspace settings
|
||||
- `session.yml`: per-session settings
|
||||
|
||||
Pipeline config lookup for CLI commands:
|
||||
|
||||
- if `--config <path>` is provided, Narratio uses that path
|
||||
- if `--config` is omitted, Narratio searches in this order:
|
||||
- `/usr/local/etc/narratio/pipeline.yml`
|
||||
- `/etc/narratio/pipeline.yml`
|
||||
|
||||
Session config lookup for CLI commands:
|
||||
|
||||
- if `--session <path>` is provided, Narratio uses that path
|
||||
- if `--session` is omitted, Narratio searches in this order:
|
||||
- `./session.yml`
|
||||
- `/usr/local/etc/narratio/session.yml`
|
||||
- `/etc/narratio/session.yml`
|
||||
|
||||
Session template support:
|
||||
|
||||
- Narratio renders `session.yml` templates before strict YAML decode.
|
||||
- `--session-id <value>` provides the `session_id` template variable.
|
||||
- Supported placeholder forms:
|
||||
- `{{session_id}}`
|
||||
- `{{ session_id }}`
|
||||
- unresolved template placeholders fail with a clear error.
|
||||
- strict YAML validation still runs after rendering.
|
||||
- concrete `session.yml` files without templates remain fully supported.
|
||||
|
||||
Optional secrets-from-files config:
|
||||
|
||||
- `pipeline.secrets.env_dir` may point to a directory of secret files
|
||||
- each top-level file with an env-var-style name is loaded as an environment variable:
|
||||
- file name = env var name
|
||||
- file contents = env var value (trailing newline/CRLF trimmed)
|
||||
- process environment wins: existing env vars are not overwritten
|
||||
- if configured, Narratio fails fast when `env_dir` is missing/unreadable
|
||||
- relative `env_dir` values resolve from Narratio’s current working directory
|
||||
|
||||
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
||||
|
||||
## Storage And Archive Foundations
|
||||
|
||||
Narratio now includes configuration and path-model foundations for archive support, plus implemented prepare-stage S3 audio input.
|
||||
|
||||
Implemented foundations:
|
||||
|
||||
- `pipeline.storage.s3` config shape (`bucket`, `root_prefix`, `region`, `endpoint`, `force_path_style`, `access_key_id_env`, `secret_access_key_env`)
|
||||
- `pipeline.spool` config shape (`root`, `delete_audio_after_archive`)
|
||||
- `pipeline.archive` config shape (`enabled`, `upload_run`, `promote_artifacts`)
|
||||
- promotion-rule validation (`from`/`to` required, relative-only paths, traversal rejected)
|
||||
- `session.campaign` requirement for campaign-aware path construction
|
||||
- optional `session.inputs.audio_s3.prefix` modeling and prepare-stage S3 audio download
|
||||
- run ID generation and S3/local path helper foundations
|
||||
- manifest run/path identity fields
|
||||
|
||||
Current defaults:
|
||||
|
||||
- `pipeline.storage.s3.root_prefix`: `dnd`
|
||||
- `pipeline.storage.s3.access_key_id_env`: `OBJECT_STORAGE_KEY_ID`
|
||||
- `pipeline.storage.s3.secret_access_key_env`: `OBJECT_STORAGE_KEY`
|
||||
- `pipeline.workspace.cleanup_after_archive`: `false`
|
||||
- `pipeline.spool.root`: `/var/spool/narratio`
|
||||
- `pipeline.spool.delete_audio_after_archive`: `false`
|
||||
- `pipeline.archive.enabled`: `true`
|
||||
- `pipeline.archive.upload_run`: `true`
|
||||
- default `pipeline.archive.promote_artifacts`:
|
||||
- `transcripts/trimmed.json` -> `transcripts/trimmed.json` (`required: true`)
|
||||
- `artifacts/session_recap.md` -> `artifacts/session_recap.md` (`required: true`)
|
||||
|
||||
Current boundaries:
|
||||
|
||||
- local development audio (`audio_dir` / `audio_files`) still works
|
||||
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
|
||||
- real S3-compatible backend now exists in the storage adapter package
|
||||
- storage backend tests use fake storage and do not require live S3
|
||||
- archive uploads successful run records under `runs/{run_id}/`
|
||||
- archive does not upload local audio by default
|
||||
- archive uploads promoted outputs to session-level keys using `archive.promote_artifacts`
|
||||
- archive uploads `current/manifest.json`
|
||||
- archive uploads `current/run_id.txt` last as the effective commit marker
|
||||
- required missing promotions fail archive
|
||||
- optional missing promotions are skipped and recorded
|
||||
- cleanup remains conservative and opt-in:
|
||||
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory after successful archive commit
|
||||
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir after successful archive commit
|
||||
- cleanup executes only after all selected stages for the command invocation succeed
|
||||
- cleanup does not run for failed, incomplete, skipped, or unarchived runs
|
||||
- local development `audio_dir`/`audio_files` source inputs are never deleted by spool cleanup
|
||||
- S3 credentials are resolved from configured env-var names when both are present; if either is missing, Narratio falls back to the AWS SDK default credential chain
|
||||
|
||||
S3 input details and current boundaries are documented in [docs/s3-audio-input.md](docs/s3-audio-input.md).
|
||||
|
||||
## Remote Storage Backend
|
||||
|
||||
Narratio includes an object-store backend layer for future prepare/archive work:
|
||||
|
||||
- `List(ctx, prefix)`
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Implemented backends:
|
||||
|
||||
- fake storage backend for deterministic tests
|
||||
- S3-compatible backend built from `pipeline.storage.s3`
|
||||
|
||||
Key invariant:
|
||||
|
||||
- callers pass full bucket-relative object keys
|
||||
- storage backends do not prepend `root_prefix` and do not infer session/campaign paths
|
||||
|
||||
Current boundary:
|
||||
|
||||
- `prepare` uses `List` + `Download` through the backend when `session.inputs.audio_s3` is configured
|
||||
- `archive` uses `Upload` through the backend for successful run-record uploads
|
||||
- `archive` also uses `Upload` for promotion writes and current pointers
|
||||
- no failed or incomplete runs are uploaded
|
||||
- local audio is not re-uploaded by default
|
||||
|
||||
Archive run-upload details and boundaries are documented in [docs/archive-storage.md](docs/archive-storage.md).
|
||||
|
||||
## Canonical Stage Order
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
## Transcript Tiers
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output
|
||||
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
## Seriatim Configuration
|
||||
|
||||
`pipeline.seriatim` configures the Seriatim subprocess adapter used by `merge`, `normalize`, and `trim`.
|
||||
|
||||
Minimal behavior:
|
||||
|
||||
- `pipeline.seriatim` may be omitted entirely.
|
||||
- when omitted, Narratio defaults to:
|
||||
- `binary: seriatim`
|
||||
- `timeout: 10m`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `coalesce_gap: 3.0`
|
||||
- `report: true`
|
||||
|
||||
Optional overrides in `pipeline.seriatim` continue to work, including explicit binary paths and advanced `env` tuning values.
|
||||
|
||||
## Audita Configuration
|
||||
|
||||
`pipeline.audita` configures the real Audita subprocess adapter used by `polish`.
|
||||
|
||||
Minimal behavior:
|
||||
|
||||
- `pipeline.audita` may be omitted entirely.
|
||||
- when omitted, Narratio defaults to:
|
||||
- `binary: audita`
|
||||
- `timeout: 3h`
|
||||
- `report: true`
|
||||
|
||||
Optional:
|
||||
|
||||
- `llm_api_key_env` (when set, Narratio requires that env var and passes it to Audita as `AUDITA_LLM_API_KEY`)
|
||||
- `modules` override list (when empty/omitted, Narratio does not pass `--modules`)
|
||||
- `base_url` (when omitted, Narratio does not pass `--base-url`; Audita runtime defaults/config may apply)
|
||||
- `model` (when omitted, Narratio does not pass `--model`; Audita runtime defaults/config may apply)
|
||||
- `transcript_description`
|
||||
- `config_path`
|
||||
- `output_schema` (`bare-segments` or `audita-v1`)
|
||||
- `work_dir_retention` (`always`, `auto`, or `never`)
|
||||
- `total_llm_concurrency` (> 0 when provided)
|
||||
- `proposal_llm_concurrency` (> 0 when provided)
|
||||
- `validation_model`
|
||||
- `validation_llm_concurrency` (> 0 when provided)
|
||||
- `report` (defaults to `true`)
|
||||
|
||||
Narratio passes only configured optional Audita flags. Omitted optional values are left to Audita runtime defaults/config.
|
||||
|
||||
## Normalize Configuration
|
||||
|
||||
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
Allowed `normalize.output_schema` values:
|
||||
|
||||
- `seriatim-minimal`
|
||||
- `seriatim-intermediate`
|
||||
- `seriatim-full`
|
||||
|
||||
`normalize.output_path` is treated as session-workdir-relative when not absolute.
|
||||
|
||||
Normalize stage behavior summary:
|
||||
|
||||
- normalize runs after `polish` and before `trim`
|
||||
- normalize resolves `transcripts/processed.json`
|
||||
- normalize runs Seriatim `normalize` to produce `transcripts/normalized.json`
|
||||
- normalize diagnostics are written to:
|
||||
- `artifacts/seriatim.normalize.report.json` (when enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## Trim Configuration
|
||||
|
||||
`pipeline.trim` is optional. If omitted, no trim config is loaded. If `trim.enabled` is omitted, it defaults to `false`.
|
||||
|
||||
When `trim.enabled: true`:
|
||||
|
||||
- `trim.output_path` is required
|
||||
- `trim.bounds.prompt_id` is required
|
||||
- `trim.bounds.transcript_input_name` is required
|
||||
- `trim.bounds.output_path` is required
|
||||
- `trim.bounds.timeout` must be a valid Go duration when provided
|
||||
- `trim.bounds.render_debug: true` requires `trim.bounds.render_output_path`
|
||||
- `trim.bounds.profile_id` may be empty to use the prompt default profile
|
||||
|
||||
Trim paths are treated as session-workdir-relative when not absolute.
|
||||
|
||||
Example trim config:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```sh
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Trim behavior summary:
|
||||
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete
|
||||
`session.yml` files or their explicit command-line alternatives.
|
||||
|
||||
- trim discovers and validates `transcripts/normalized.json`
|
||||
- trim uses Scriptorium bounds (`dnd_session.bounds` by example config) to produce `artifacts/session_bounds.json`
|
||||
- bounds IDs are validated against the same normalized transcript ID space that Seriatim trim will consume
|
||||
- trim converts bounds to Seriatim keep selector (for example `10-868`) and runs Seriatim trim
|
||||
- if trim is disabled, Narratio copies normalized transcript to trimmed transcript and records `trim_action=copy_disabled`
|
||||
## Documentation
|
||||
|
||||
Trim outputs and diagnostics:
|
||||
- [CLI reference](docs/cli.md) — commands, arguments, flags, and invocation
|
||||
behavior.
|
||||
- [Configuration](docs/config.md) — discovery, fields, defaults, and
|
||||
validation.
|
||||
- [Operations](docs/operations.md) — runtime workflow, state, publishing,
|
||||
recovery, and cleanup.
|
||||
- [Troubleshooting](docs/troubleshooting.md) — symptom-driven diagnosis and
|
||||
safe remedies.
|
||||
- [Integration contracts](docs/integrations/) — external tools, formats, and
|
||||
compatibility expectations.
|
||||
- [Maintained examples](examples/README.md) — complete copyable configuration
|
||||
and input files.
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
## Maintainer Documentation
|
||||
|
||||
Render-debug files are diagnostics and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## Scriptorium Configuration
|
||||
|
||||
`pipeline.scriptorium` is optional. When present, Narratio validates and uses it for analyze-stage artifact generation.
|
||||
|
||||
Key points:
|
||||
|
||||
- `scriptorium.binary` defaults to `scriptorium` when section is present
|
||||
- `scriptorium.config_path` is optional
|
||||
- `scriptorium.timeout` defaults to `10m` when omitted
|
||||
- `scriptorium.render_debug` enables render diagnostics globally
|
||||
- artifacts are configured under `scriptorium.artifacts` (map shape supports multiple artifacts)
|
||||
- enabled artifacts require `prompt_id` and `output_path`
|
||||
- artifact `render_debug` may override global render setting
|
||||
- `vars` currently support boolean and string values
|
||||
|
||||
Example `session_recap` artifact definition:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: "scriptorium"
|
||||
config_path: "/etc/scriptorium/config.yml"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
profile_id: "local-quality" # optional
|
||||
output_path: "artifacts/session_recap.md"
|
||||
timeout: "10m"
|
||||
# render_debug: true # optional per-artifact override
|
||||
|
||||
inputs:
|
||||
transcript:
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
path: "" # optional; set when available
|
||||
required: false
|
||||
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
```
|
||||
|
||||
Prompt IDs and profile IDs are configuration values. They are not hardcoded in analyze-stage logic.
|
||||
|
||||
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
||||
|
||||
If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs.
|
||||
|
||||
## Scriptorium Runtime Behavior
|
||||
|
||||
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
||||
|
||||
- generation: `scriptorium run`
|
||||
- diagnostics/testing: `scriptorium render --format json` when `render_debug` is enabled
|
||||
|
||||
For the initial implementation, only `session_recap` generation is supported.
|
||||
|
||||
Analyze-stage session recap behavior:
|
||||
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
||||
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
|
||||
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
|
||||
- missing trimmed transcript fails clearly and advises running trim stage first
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains supported for advanced/debug use cases
|
||||
- optionally includes `previous_recap` when configured and resolvable
|
||||
- omits optional previous recap when unavailable
|
||||
- fails if required inputs are missing
|
||||
- validates output file exists and is non-empty
|
||||
|
||||
Expected session output paths:
|
||||
|
||||
- `artifacts/session_recap.md`
|
||||
- `logs/scriptorium.session_recap.stdout.log`
|
||||
- `logs/scriptorium.session_recap.stderr.log`
|
||||
- `config/scriptorium.session_recap.generated.yml`
|
||||
- `artifacts/session_recap.render.json` when render diagnostics are enabled
|
||||
|
||||
## Examples
|
||||
|
||||
Starter files:
|
||||
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/pipeline.audita-overrides.yml`
|
||||
- `examples/session.minimal.yml`
|
||||
- `examples/session.template.yml`
|
||||
- `examples/speakers.yml`
|
||||
|
||||
## Commands
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Plan a run:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio plan --session examples/session.minimal.yml
|
||||
```
|
||||
|
||||
Use `--config <path>` to override default pipeline lookup when needed.
|
||||
|
||||
Run with a discoverable session template:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run --session-id 2026-04-04
|
||||
```
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
||||
```
|
||||
|
||||
Run analyze only:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
||||
```
|
||||
|
||||
Resume with a template session ID:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio resume --config examples/pipeline.minimal.yml --session examples/session.template.yml --session-id 2026-04-04
|
||||
```
|
||||
|
||||
## Operational Note
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime config change, rerun the appropriate upstream stages before relying on downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary/autocorrect/speaker-context changes: rerun at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim bounds prompt/profile/config changes: rerun at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes: rerun `analyze`
|
||||
|
||||
## Roadmap
|
||||
|
||||
Near-term roadmap:
|
||||
|
||||
- extend analyze to additional configured artifacts
|
||||
- support workflows where later artifacts consume earlier generated artifacts
|
||||
- keep orchestration explicit without a generic DAG engine
|
||||
- implement archive and notify backends
|
||||
- [Development guide](docs/development.md) — first-read orientation and
|
||||
task-specific reading routes.
|
||||
- [Internal overview](docs/internal/overview.md) — implemented component map.
|
||||
- [Architecture](docs/policy/architecture.md) — normative boundaries and
|
||||
invariants.
|
||||
- [Documentation policy](docs/policy/documentation.md) — canonical ownership
|
||||
and maintenance rules.
|
||||
- [Testing policy](docs/policy/testing.md) — test value, boundaries, and
|
||||
sufficiency.
|
||||
|
||||
501
architecture.md
501
architecture.md
@@ -1,501 +0,0 @@
|
||||
# Narratio Architecture
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
`narratio` is a Go orchestrator for D&D session processing. It runs a stage-based local pipeline from audio input through transcript processing and artifact generation, with manifest-based skip/force/resume behavior.
|
||||
|
||||
Narratio integrates with Scriptorium through the **public CLI** (`scriptorium run` and `scriptorium render`) via synchronous subprocess execution.
|
||||
|
||||
## 2. Current Status
|
||||
|
||||
Implemented:
|
||||
|
||||
- strict `pipeline.yml` + `session.yml` loading with strict YAML field checking (`KnownFields(true)`)
|
||||
- local workspace/session layout, lock file handling, artifact path helpers, checksums, and atomic writes
|
||||
- manifest store and stage status transitions for resumable runs
|
||||
- real `prepare`, `transcribe`, `merge`, and `polish` stages
|
||||
- real WhisperX HTTP adapter
|
||||
- real Seriatim subprocess adapter
|
||||
- real Audita subprocess adapter
|
||||
- real Scriptorium subprocess adapter
|
||||
- real `normalize` stage producing `transcripts/normalized.json`
|
||||
- real `trim` stage producing `transcripts/trimmed.json`
|
||||
- real `analyze` stage for initial `session_recap` generation
|
||||
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
||||
- storage/archive configuration and validation foundations for:
|
||||
- `pipeline.storage.s3`
|
||||
- `pipeline.spool`
|
||||
- `pipeline.archive` promotion rules
|
||||
- `session.inputs.audio_s3`
|
||||
- run identity and path-model foundations:
|
||||
- run ID generation (`YYYYMMDDTHHMMSSZ-xxxxxxxx`)
|
||||
- S3 session/run/current key builders
|
||||
- campaign/session/run local work/spool path helpers
|
||||
- manifest run/path identity fields (`campaign`, `run_id`, local and S3 prefixes)
|
||||
- remote storage backend layer:
|
||||
- narrow object-store interface (`List`, `Download`, `Upload`, `Exists`)
|
||||
- fake storage backend for deterministic tests (no network dependency)
|
||||
- S3-compatible backend using AWS SDK v2
|
||||
- config-based object-store construction helper
|
||||
|
||||
Still placeholder/future:
|
||||
|
||||
- `notify` stage behavior
|
||||
- additional Scriptorium artifact types beyond `session_recap`
|
||||
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
||||
- generic stale detection based on input/config checksums
|
||||
|
||||
## 3. Pipeline and Stage Boundaries
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
Boundary rules:
|
||||
|
||||
- orchestration logic lives in `internal/app`
|
||||
- stage business logic lives in `internal/stage`
|
||||
- external-tool CLI construction lives in adapter packages
|
||||
- Scriptorium CLI details stay in `internal/adapters/scriptorium`
|
||||
|
||||
## 4. Scriptorium Integration Model
|
||||
|
||||
Integration mode:
|
||||
|
||||
- public CLI subprocesses only (no Scriptorium internal Go packages, no HTTP API)
|
||||
- production generation uses `scriptorium run`
|
||||
- diagnostics/testing render uses `scriptorium render --format json`
|
||||
|
||||
Run invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium run --prompt <prompt_id> --input name=path --out <output_path>
|
||||
```
|
||||
|
||||
Optional flags passed when configured:
|
||||
|
||||
- `--config <path>`
|
||||
- `--profile <profile_id>`
|
||||
- repeated `--var name=value`
|
||||
- repeated `--input name=path`
|
||||
- `--timeout <duration>`
|
||||
- `--api-key-env <ENV_NAME>` when configured
|
||||
|
||||
Render invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium render --prompt <prompt_id> --input name=path --format json --out <render_output_path>
|
||||
```
|
||||
|
||||
Adapter behavior:
|
||||
|
||||
- always passes `--out`
|
||||
- captures stdout/stderr separately
|
||||
- writes generated invocation metadata YAML (redacted, no secrets)
|
||||
- treats exit code `0` as success
|
||||
- treats exit code `1` as failure
|
||||
- treats exit code `2` as failure with `validation_failed=true` and preserves output metadata when available
|
||||
- validates successful output files exist and are non-empty
|
||||
- does not treat non-empty stderr as failure by itself
|
||||
|
||||
## 5. Configuration Contract
|
||||
|
||||
CLI pipeline config path resolution:
|
||||
|
||||
- when `--config <path>` is provided, that path is used
|
||||
- when `--config` is omitted, Narratio searches defaults in order:
|
||||
- `/usr/local/etc/narratio/pipeline.yml`
|
||||
- `/etc/narratio/pipeline.yml`
|
||||
|
||||
CLI session config path resolution:
|
||||
|
||||
- when `--session <path>` is provided, that path is used
|
||||
- when `--session` is omitted, Narratio searches defaults in order:
|
||||
- `./session.yml`
|
||||
- `/usr/local/etc/narratio/session.yml`
|
||||
- `/etc/narratio/session.yml`
|
||||
|
||||
Session template rendering:
|
||||
|
||||
- session templates are rendered before strict YAML decode
|
||||
- `--session-id <value>` provides the `session_id` template variable
|
||||
- supported placeholders:
|
||||
- `{{session_id}}`
|
||||
- `{{ session_id }}`
|
||||
- unresolved placeholders fail clearly
|
||||
- strict `KnownFields(true)` YAML validation still applies after rendering
|
||||
- if rendered `session.session_id` conflicts with `--session-id`, load fails clearly
|
||||
|
||||
Optional pipeline secrets directory:
|
||||
|
||||
- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution
|
||||
- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed)
|
||||
- only env-var-style file names are considered; other entries are ignored
|
||||
- existing process environment values are preserved (not overwritten)
|
||||
- if configured, unreadable/missing `env_dir` fails command execution early
|
||||
- relative `env_dir` values are resolved from current working directory
|
||||
|
||||
Storage and archive foundations:
|
||||
|
||||
- `pipeline.storage.s3` is available for modeling S3 coordinates:
|
||||
- `bucket`
|
||||
- `root_prefix` (default `dnd`)
|
||||
- `region`
|
||||
- `endpoint`
|
||||
- `force_path_style` (default `false`)
|
||||
- `access_key_id_env` (default `OBJECT_STORAGE_KEY_ID`)
|
||||
- `secret_access_key_env` (default `OBJECT_STORAGE_KEY`)
|
||||
- `pipeline.spool.root` defaults to `/var/spool/narratio`
|
||||
- `pipeline.workspace.cleanup_after_archive` defaults to `false`
|
||||
- `pipeline.spool.delete_audio_after_archive` defaults to `false`
|
||||
- `pipeline.archive` is optional and defaults to:
|
||||
- `enabled: true`
|
||||
- `upload_run: true`
|
||||
- default `promote_artifacts`:
|
||||
- `transcripts/trimmed.json`
|
||||
- `artifacts/session_recap.md`
|
||||
- archive promotion rules enforce safe relative paths:
|
||||
- `from` and `to` are required
|
||||
- absolute paths are rejected
|
||||
- traversal segments such as `..` are rejected
|
||||
|
||||
Session input foundations:
|
||||
|
||||
- `session.campaign` is required
|
||||
- local audio remains supported through `session.inputs.audio_dir` or `session.inputs.audio_files`
|
||||
- optional S3 audio input shape is `session.inputs.audio_s3.prefix`
|
||||
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
|
||||
- when `audio_s3` is configured, `prepare` lists and downloads `.flac` objects through the object-store backend
|
||||
|
||||
Cross-config validation scope:
|
||||
|
||||
- `pipeline.storage.s3.bucket` is required only when an S3-dependent feature is explicitly configured (for current foundations, that includes `session.inputs.audio_s3`, and archive upload intent when using `storage.backend: s3`)
|
||||
- no AWS credential values are stored in Narratio config; only env-var names are configured
|
||||
- when both configured credential env vars resolve to non-empty values, the S3 backend uses them as static credentials
|
||||
- when either configured credential value is missing, the S3 backend falls back to the AWS SDK default credential chain
|
||||
|
||||
Remote object-store backend scope:
|
||||
|
||||
- remote storage APIs are isolated to `internal/adapters/storage`
|
||||
- AWS SDK types remain contained within the S3 backend implementation package
|
||||
- S3 key/session path semantics remain outside the backend, with this invariant:
|
||||
- callers pass full bucket-relative object keys
|
||||
- backend methods do not prepend `root_prefix` or infer campaign/session/run paths
|
||||
- `prepare` now uses object-store `List` and `Download` for S3 audio input
|
||||
- `archive` now uses object-store `Upload` for successful run-record upload under the run prefix
|
||||
- `archive` now uses object-store `Upload` for promoted outputs and current pointers
|
||||
|
||||
Prepare S3 audio behavior (implemented):
|
||||
|
||||
- compute session prefix as `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||
- resolve `session.inputs.audio_s3.prefix` under that session prefix
|
||||
- list objects under the computed audio prefix and filter `.flac` keys
|
||||
- fail clearly when no `.flac` objects are found
|
||||
- download selected objects to spool audio path:
|
||||
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||
- materialize audio files into workdir audio path:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
|
||||
- record S3 provenance in manifest input records (bucket/key/metadata/local paths/checksum)
|
||||
- no AWS SDK types are used in stage code; storage implementation details stay in storage adapter packages
|
||||
|
||||
Archive publishing behavior (implemented):
|
||||
|
||||
- `archive` verifies prerequisite stage success before upload:
|
||||
- `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`
|
||||
- only successful/completed runs are uploaded
|
||||
- uploaded run record destination is:
|
||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
|
||||
- uploaded existing local paths include:
|
||||
- `inputs/`, `transcripts/`, `artifacts/`, optional `reports/`, `config/`, `logs/`, and `manifest.json`
|
||||
- local `audio/` is intentionally excluded from upload by default
|
||||
- file upload order is deterministic (sorted relative paths)
|
||||
- `archive.enabled: false` and `archive.upload_run: false` skip upload cleanly
|
||||
- stage metadata records non-secret upload context:
|
||||
- run upload details, promoted output details, current manifest key, current pointer key
|
||||
- no secrets, transcript contents, prompt contents, or environment dumps
|
||||
- promotion rules:
|
||||
- `from` resolves from local workdir
|
||||
- `to` resolves under session-level S3 root
|
||||
- missing required source fails archive
|
||||
- missing optional source is skipped and recorded
|
||||
- default promoted outputs:
|
||||
- `transcripts/trimmed.json`
|
||||
- `artifacts/session_recap.md`
|
||||
- current pointers:
|
||||
- `current/manifest.json` uploaded after run upload and promotions
|
||||
- `current/run_id.txt` uploaded last with `{run_id}\n`
|
||||
- `current/run_id.txt` is the effective commit marker
|
||||
- if promotion or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`
|
||||
- failed/incomplete runs remain local and are not uploaded
|
||||
- post-archive local cleanup (implemented, opt-in):
|
||||
- cleanup runs only after archive succeeded and wrote `current/run_id.txt`
|
||||
- cleanup is executed after all selected stages in the command invocation succeed (for example, a later `notify` failure leaves local files intact)
|
||||
- `pipeline.spool.delete_audio_after_archive: true` removes only `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||
- `pipeline.workspace.cleanup_after_archive: true` removes only `{workspace.root}/work/{campaign}/{session_id}/{run_id}/`
|
||||
- cleanup does not run when archive is skipped/disabled/fails or when run upload is disabled
|
||||
- local development `audio_dir`/`audio_files` inputs are never removed by spool cleanup
|
||||
|
||||
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
||||
|
||||
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
||||
|
||||
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
||||
|
||||
`pipeline.audita` drives the real Audita subprocess adapter for the `polish` stage.
|
||||
|
||||
Audita defaulted fields:
|
||||
|
||||
- `binary` defaults to `audita`
|
||||
- `timeout` defaults to `3h`
|
||||
- `report` defaults to `true`
|
||||
|
||||
Audita optional fields:
|
||||
|
||||
- `llm_api_key_env` (enforced only when configured)
|
||||
- `modules` override list (when omitted/empty, Narratio does not pass `--modules`)
|
||||
- `base_url` (when omitted, Narratio does not pass `--base-url`)
|
||||
- `model` (when omitted, Narratio does not pass `--model`)
|
||||
- `transcript_description`
|
||||
- `config_path`
|
||||
- `output_schema` (`bare-segments` or `audita-v1`)
|
||||
- `work_dir_retention` (`always`, `auto`, `never`)
|
||||
- `total_llm_concurrency` (> 0 when provided)
|
||||
- `proposal_llm_concurrency` (> 0 when provided)
|
||||
- `validation_model`
|
||||
- `validation_llm_concurrency` (> 0 when provided)
|
||||
- `report` override
|
||||
|
||||
Narratio passes only configured optional Audita flags; omitted optional values defer to Audita runtime defaults/config.
|
||||
|
||||
Seriatim defaults:
|
||||
|
||||
- `pipeline.seriatim` may be omitted
|
||||
- `binary` defaults to `seriatim`
|
||||
- `timeout` defaults to `10m`
|
||||
- `output_schema` defaults to `seriatim-intermediate`
|
||||
- `coalesce_gap` defaults to `3.0`
|
||||
- `report` defaults to `true`
|
||||
|
||||
When `pipeline.normalize` is omitted, defaults are applied:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
When `pipeline.normalize` is present:
|
||||
|
||||
- `output_path` must be non-empty
|
||||
- `output_schema` must be one of `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`
|
||||
- relative `output_path` values are session-workdir-relative paths
|
||||
- Seriatim binary settings still come from `pipeline.seriatim`
|
||||
|
||||
When `pipeline.trim` is present:
|
||||
|
||||
- `enabled` is optional and defaults to `false` when omitted
|
||||
- relative `output_path`, `bounds.output_path`, and `bounds.render_output_path` values are session-workdir-relative paths
|
||||
- do not store secrets in trim config values
|
||||
|
||||
When `pipeline.trim.enabled: true`:
|
||||
|
||||
- `output_path` is required and non-empty
|
||||
- `bounds.prompt_id` is required and non-empty
|
||||
- `bounds.transcript_input_name` is required and non-empty
|
||||
- `bounds.output_path` is required and non-empty
|
||||
- `bounds.timeout` must parse as a Go duration when provided
|
||||
- `bounds.render_debug: true` requires non-empty `bounds.render_output_path`
|
||||
- `bounds.profile_id` may be empty to use the prompt default profile
|
||||
- prompt IDs are config values, not hardcoded stage logic
|
||||
|
||||
When `pipeline.scriptorium` is present:
|
||||
|
||||
- `binary` defaults to `scriptorium` when omitted
|
||||
- `config_path` is optional; when provided it must be non-empty
|
||||
- `timeout` is optional; when provided it must parse as a Go duration
|
||||
- default `timeout` is `10m`
|
||||
- unknown YAML fields fail strict decode
|
||||
|
||||
Artifacts are configured as a map under `pipeline.scriptorium.artifacts` so multiple artifacts are possible in the config shape.
|
||||
|
||||
For each artifact definition:
|
||||
|
||||
- `enabled: true` requires non-empty `prompt_id`
|
||||
- `enabled: true` requires non-empty `output_path`
|
||||
- `timeout` must parse as Go duration when present
|
||||
- optional per-artifact `render_debug` may override global `scriptorium.render_debug`
|
||||
- `inputs` are named and each input requires non-empty `source`
|
||||
- inputs may be optional (`required: false`)
|
||||
- `vars` values currently support `string` and `bool`
|
||||
|
||||
Prompt IDs and profile IDs are configuration values, not hardcoded stage logic.
|
||||
|
||||
Trim config shape:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```
|
||||
|
||||
## 6. Transcript Tiers
|
||||
|
||||
Narratio currently produces and uses four transcript tiers:
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output (includes pre/post-game content)
|
||||
- `transcripts/normalized.json`: normalized transcript generated by Seriatim normalize
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
Trim reads `transcripts/normalized.json`, validates bounds IDs against that same transcript ID space, and writes `transcripts/trimmed.json`.
|
||||
|
||||
## 7. Normalize Stage (Current Implementation)
|
||||
|
||||
Normalize stage behavior:
|
||||
|
||||
- stage order position: after `polish` and before `trim`
|
||||
- discovers processed transcript from manifest polish outputs (`transcript_processed`) when present, else `work/<session_id>/transcripts/processed.json`
|
||||
- validates processed transcript JSON shape (`segments` array required)
|
||||
- runs Seriatim `normalize` to produce normalized transcript
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- validates normalize report JSON when enabled
|
||||
|
||||
Expected normalize outputs and diagnostics:
|
||||
|
||||
- `transcripts/normalized.json`
|
||||
- `artifacts/seriatim.normalize.report.json` (when normalize report is enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## 8. Trim Stage (Current Implementation)
|
||||
|
||||
Trim stage behavior:
|
||||
|
||||
- stage order position: after `normalize` and before `analyze`
|
||||
- discovers normalized transcript from manifest normalize outputs (`transcript_normalized`) when present, else `work/<session_id>/transcripts/normalized.json`
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- when `trim.enabled: false` (or trim config omitted), deterministically copies normalized transcript to `transcripts/trimmed.json` and records `trim_action=copy_disabled`
|
||||
- when `trim.enabled: true`:
|
||||
- runs Scriptorium bounds prompt using configured `trim.bounds.prompt_id`
|
||||
- writes bounds output to configured path (typically `artifacts/session_bounds.json`)
|
||||
- parses and validates bounds output against the same normalized transcript being trimmed
|
||||
- converts bounds range to Seriatim keep selector (for example `10-868`)
|
||||
- runs Seriatim `trim` to produce `transcripts/trimmed.json`
|
||||
- supports no-trim bounds actions (`none`/`copy`) by copying normalized transcript unchanged
|
||||
- validates trimmed transcript JSON shape (`segments` array required)
|
||||
|
||||
Expected trim outputs and diagnostics:
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs when enabled:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
|
||||
Render-debug files are diagnostics. They are recorded in stage metadata/log/config refs and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## 9. Analyze Stage (Current Implementation)
|
||||
|
||||
The current real analyze implementation supports only `scriptorium.artifacts.session_recap`.
|
||||
|
||||
Behavior:
|
||||
|
||||
- if `pipeline.scriptorium` is missing, analyze returns a skipped result with metadata
|
||||
- if no Scriptorium artifacts are enabled, analyze returns a skipped result with metadata
|
||||
- if enabled artifacts exist but `session_recap` is not enabled, analyze fails clearly
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- `session_recap` should use `trimmed_transcript` input (`transcripts/trimmed.json`) for in-universe recap generation
|
||||
- `trimmed_transcript` input is resolved from manifest (`trim` output kind `transcript_trimmed`) when available, otherwise fallback path `work/<session_id>/transcripts/trimmed.json`
|
||||
- `normalized_transcript` input is resolved from manifest (`normalize` output kind `transcript_normalized`) when available, otherwise fallback path `work/<session_id>/transcripts/normalized.json`
|
||||
- `processed_transcript` input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work/<session_id>/transcripts/processed.json`
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains available for advanced/debug use cases
|
||||
- transcript inputs are validated as JSON with top-level `segments` array
|
||||
- configured inputs are resolved by source
|
||||
- optional `previous_recap` is omitted when unavailable
|
||||
- required `previous_recap` fails before invocation when unavailable
|
||||
- vars are built from config + session metadata
|
||||
- `render_debug` controls pre-run `scriptorium render` diagnostics
|
||||
- render failure stops stage before production run
|
||||
- render output is validated as JSON
|
||||
- production call uses Scriptorium adapter `RunArtifact`
|
||||
- successful run output must exist and be non-empty
|
||||
- missing `trimmed_transcript` input for configured `trimmed_transcript` source fails clearly with guidance to run trim stage first
|
||||
- manifest records output refs, logs, generated config paths, and non-secret provenance metadata
|
||||
|
||||
## 10. Session Recap Paths
|
||||
|
||||
Current expected paths for `session_recap`:
|
||||
|
||||
- artifact output: `artifacts/session_recap.md`
|
||||
- run stdout log: `logs/scriptorium.session_recap.stdout.log`
|
||||
- run stderr log: `logs/scriptorium.session_recap.stderr.log`
|
||||
- run generated invocation/config: `config/scriptorium.session_recap.generated.yml`
|
||||
- render output (when enabled): `artifacts/session_recap.render.json`
|
||||
- render stdout log: `logs/scriptorium.session_recap.render.stdout.log`
|
||||
- render stderr log: `logs/scriptorium.session_recap.render.stderr.log`
|
||||
- render generated invocation/config: `config/scriptorium.session_recap.render.generated.yml`
|
||||
|
||||
## 11. Security and Privacy
|
||||
|
||||
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
||||
- if API-key integration is configured, pass env var names only (never raw key values)
|
||||
- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs
|
||||
- avoid logging transcript content or rendered prompt content by default
|
||||
- treat generated artifacts and logs as potentially sensitive session material
|
||||
|
||||
## 12. Operational Caveat (Pre-Stale-Detection)
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime configuration change (for example glossary files, prompt IDs, profile IDs, or relevant pipeline settings), rerun the appropriate prior stages to refresh downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary or autocorrect changes usually require rerunning at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim prompt/profile changes require rerunning at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes require rerunning `analyze`
|
||||
|
||||
## 13. Roadmap
|
||||
|
||||
Planned next steps:
|
||||
|
||||
- extend analyze beyond `session_recap` to additional configured artifacts
|
||||
- support artifact inputs that consume prior generated artifacts
|
||||
- keep this composable without adding a generic DAG engine in the near term
|
||||
- implement real `archive` backend behavior
|
||||
- implement real `notify` backend behavior
|
||||
- add checksum-based stale detection and stale transitions
|
||||
|
||||
Architectural invariants remain:
|
||||
|
||||
- strict config decoding/validation
|
||||
- manifest-driven run control
|
||||
- clear stage/adapter separation
|
||||
- configuration-driven prompt/profile/input/vars/output mapping
|
||||
- Scriptorium integration through public CLI subprocess contract
|
||||
@@ -1,119 +0,0 @@
|
||||
# Archive Storage
|
||||
|
||||
This document describes implemented archive-stage publish behavior.
|
||||
|
||||
## S3 Paths
|
||||
|
||||
Session root:
|
||||
|
||||
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||
|
||||
Run prefix:
|
||||
|
||||
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
|
||||
|
||||
## Scope
|
||||
|
||||
Implemented:
|
||||
|
||||
- archive uploads successful run records to remote object storage through the storage backend abstraction.
|
||||
- archive uploads configured promoted outputs to session-level keys.
|
||||
- archive uploads `current/manifest.json`.
|
||||
- archive uploads `current/run_id.txt` last as the effective commit marker.
|
||||
- optional post-archive local cleanup:
|
||||
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory
|
||||
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir
|
||||
- tests use fake storage and do not require live S3.
|
||||
|
||||
Future work:
|
||||
|
||||
- `notify` stage behavior
|
||||
- stale detection
|
||||
- optional future source-audio upload mode
|
||||
- additional artifact generation beyond current implemented set
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Archive verifies these stages succeeded before upload:
|
||||
|
||||
- `prepare`
|
||||
- `transcribe`
|
||||
- `merge`
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `analyze`
|
||||
|
||||
If any prerequisite is missing or not succeeded, archive fails and does not upload.
|
||||
Failed or incomplete runs remain local only.
|
||||
|
||||
## Run Upload
|
||||
|
||||
Archive uploads existing files from the run workdir when present:
|
||||
|
||||
- `inputs/`
|
||||
- `transcripts/`
|
||||
- `artifacts/`
|
||||
- `reports/` (optional)
|
||||
- `config/`
|
||||
- `logs/`
|
||||
- `manifest.json`
|
||||
|
||||
Relative paths are preserved under `runs/{run_id}/`.
|
||||
|
||||
## Promotion Rules
|
||||
|
||||
Archive applies `archive.promote_artifacts` in config order.
|
||||
|
||||
Rule behavior:
|
||||
|
||||
- `from`: local workdir-relative source path
|
||||
- `to`: session-root-relative destination key
|
||||
- `required: true`: missing source fails archive
|
||||
- `required: false`: missing source is skipped and recorded
|
||||
|
||||
Default promoted outputs:
|
||||
|
||||
- `transcripts/trimmed.json`
|
||||
- `artifacts/session_recap.md`
|
||||
|
||||
## Current Pointers
|
||||
|
||||
Archive writes:
|
||||
|
||||
1. `current/manifest.json` (after run upload + promotions)
|
||||
2. `current/run_id.txt` last
|
||||
|
||||
`current/run_id.txt` contains exactly:
|
||||
|
||||
- `{run_id}` plus trailing newline
|
||||
|
||||
Writing `current/run_id.txt` last makes it the effective commit marker for published session state.
|
||||
|
||||
If any required run upload, promotion upload, or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`.
|
||||
Cleanup runs only after this commit-marker write has succeeded.
|
||||
|
||||
## Audio Upload Policy
|
||||
|
||||
Archive does not upload local `audio/` by default.
|
||||
Original audio is expected at the session-level audio prefix and is not duplicated under `runs/{run_id}/`.
|
||||
|
||||
## Config Controls
|
||||
|
||||
- `archive.enabled: false` skips archive cleanly.
|
||||
- `archive.upload_run: false` skips run upload cleanly.
|
||||
- both skip cases also skip post-archive local cleanup.
|
||||
|
||||
## Metadata
|
||||
|
||||
Archive stage metadata includes non-secret upload context (for example):
|
||||
|
||||
- `s3_bucket`
|
||||
- `s3_run_prefix`
|
||||
- run upload counts/paths
|
||||
- promoted upload counts/paths
|
||||
- skipped optional promotions
|
||||
- `current_manifest_key`
|
||||
- `current_run_id_key`
|
||||
- `current_pointer_written`
|
||||
- `audio_upload_skipped`
|
||||
301
docs/cli.md
Normal file
301
docs/cli.md
Normal file
@@ -0,0 +1,301 @@
|
||||
# CLI Reference
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
This runs the canonical full pipeline for session `2026-04-04`.
|
||||
|
||||
## Command Overview
|
||||
|
||||
Top-level commands:
|
||||
|
||||
- `run <session_id>`: run full stage order.
|
||||
- `run-stage <stage> <session_id>`: run one stage.
|
||||
- `analyze <session_id>`: force-run analyze.
|
||||
- `publish <session_id>`: force-run publish.
|
||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||
- `session <subcommand>`: session helper commands.
|
||||
|
||||
Session subcommands:
|
||||
|
||||
- `session init <session_id>`
|
||||
- `session plan <session_id>`
|
||||
- `session validate <session_id>`
|
||||
- `session status <session_id>`
|
||||
- `session restore <session_id>`
|
||||
- `session artifacts <session_id>`
|
||||
- `session locks <session_id>`
|
||||
- `session locks add <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>`
|
||||
|
||||
## Common Config Flags
|
||||
|
||||
Most session-aware commands accept:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>`
|
||||
- `--campaign-file <campaign.yml>`
|
||||
- `--session <session.yml>`
|
||||
- `--session-id <session_id>`
|
||||
- `--previous-session-id <session_id>`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||
- `--session` is not used by `session init`.
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
|
||||
## Session ID Input Rules
|
||||
|
||||
Session-aware commands accept one of these forms:
|
||||
|
||||
- positional session ID: `... <session_id>`
|
||||
- compatibility flag: `... --session-id <session_id>`
|
||||
|
||||
When both are present, command parsing requires an exact match.
|
||||
|
||||
Commands with additional positionals keep their command-specific order:
|
||||
|
||||
- `run-stage <stage> <session_id>` or `run-stage <stage> --session-id <session_id>`
|
||||
- `session locks add <session_id> <source>` or `session locks add --session-id <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>` or `session locks remove --session-id <session_id> <source>`
|
||||
|
||||
## Command Reference
|
||||
|
||||
### `run`
|
||||
|
||||
```bash
|
||||
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- evaluates full stage order;
|
||||
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||
resume check finds its durable result obsolete;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes session and run manifests.
|
||||
|
||||
### `run-stage`
|
||||
|
||||
```bash
|
||||
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Valid stage names:
|
||||
|
||||
- `prepare`
|
||||
- `transcribe`
|
||||
- `merge`
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `extract`
|
||||
- `render`
|
||||
- `analyze`
|
||||
- `publish`
|
||||
- `notify`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--artifacts` is accepted only for `analyze` and `publish` stage targets.
|
||||
|
||||
### `analyze`
|
||||
|
||||
```bash
|
||||
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
### `publish`
|
||||
|
||||
```bash
|
||||
narratio publish <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
### `clean`
|
||||
|
||||
```bash
|
||||
narratio clean <session_id> [--dry-run] [--clear-cache] [...common config flags]
|
||||
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- session mode removes the selected session's local work and spool state;
|
||||
- `--all` removes all local session work and spool state;
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
|
||||
post-publish cleanup behavior.
|
||||
|
||||
### `session plan`
|
||||
|
||||
```bash
|
||||
narratio session plan <session_id> [--force] [...common config flags]
|
||||
```
|
||||
|
||||
Validates config, prepares local workdir layout, and prints run/skip decisions for each stage.
|
||||
|
||||
### `session validate`
|
||||
|
||||
```bash
|
||||
narratio session validate <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
||||
|
||||
### `session status`
|
||||
|
||||
```bash
|
||||
narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, remote current-state and published-output status.
|
||||
|
||||
### `session init`
|
||||
|
||||
```bash
|
||||
narratio session init <session_id> --output ./session.yml [options]
|
||||
narratio session init <session_id> --remote [options]
|
||||
```
|
||||
|
||||
Required target selection:
|
||||
|
||||
- exactly one of:
|
||||
- `--output <path>`
|
||||
- `--remote`
|
||||
|
||||
Options:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||
- `--previous-session-id <id>`
|
||||
- `--date <YYYY-MM-DD>`
|
||||
- `--title <text>`
|
||||
- `--audio-dir <path>`
|
||||
- `--audio-s3-prefix <prefix>`
|
||||
- `--force`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--audio-dir` and `--audio-s3-prefix` are mutually exclusive.
|
||||
- if campaign `session_template_file` is configured, `session init` renders it.
|
||||
- generated session YAML must be concrete (no unresolved `{{ ... }}` placeholders).
|
||||
|
||||
### `session restore`
|
||||
|
||||
```bash
|
||||
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- discovers committed remote current state;
|
||||
- plans local restores;
|
||||
- writes an execution report;
|
||||
- blocks conflicting overwrites unless `--force` is set.
|
||||
|
||||
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
|
||||
default restore scope, report location, and conflict-handling workflow.
|
||||
|
||||
### `session artifacts`
|
||||
|
||||
```bash
|
||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
```
|
||||
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability.
|
||||
|
||||
### `session locks`
|
||||
|
||||
```bash
|
||||
narratio session locks <session_id> [...common config flags]
|
||||
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common config flags]
|
||||
narratio session locks remove <session_id> <source> [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode reports the effective merge of static and remote locks;
|
||||
- add/remove mutate only remote locks;
|
||||
- static locks from pipeline config cannot be removed by CLI commands.
|
||||
|
||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||
and precedence.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
- accepted on `run`, `run-stage`, `analyze`, and `publish`;
|
||||
- names must exist in `pipeline.scriptorium.artifacts`;
|
||||
- empty entries are invalid;
|
||||
- repeated names are deduplicated.
|
||||
|
||||
Effects:
|
||||
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules that source `narratio.artifact.<name>`;
|
||||
- does not filter built-in transcript/bounds or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- does not select or filter Notarius lanes.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Dry-run restore plan:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Generate a concrete session file from template/default structure:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Force publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
## Output And Exit Behavior
|
||||
|
||||
- Successful commands write their result or summary to standard output and
|
||||
exit with status `0`.
|
||||
- Command failures and invalid invocations write an error to standard error and
|
||||
exit with status `1`.
|
||||
- An unknown top-level command also prints the top-level usage summary to
|
||||
standard error.
|
||||
- `session restore --help` prints its command-specific usage and exits with
|
||||
status `0`.
|
||||
|
||||
Output is intended for operator inspection. Narratio does not currently offer
|
||||
a machine-readable CLI output mode; durable machine-readable state is recorded
|
||||
in manifests and reports described in [Operations](./operations.md).
|
||||
309
docs/config.md
Normal file
309
docs/config.md
Normal file
@@ -0,0 +1,309 @@
|
||||
# Configuration Reference
|
||||
|
||||
## Purpose
|
||||
|
||||
Narratio resolves three YAML documents:
|
||||
|
||||
- `pipeline.yml`: pipeline/runtime settings
|
||||
- `campaign.yml`: campaign identity and stable input defaults
|
||||
- `session.yml`: session identity, metadata, and audio source selection
|
||||
|
||||
## Discovery and Selection
|
||||
|
||||
### `pipeline.yml`
|
||||
|
||||
When `--config` is omitted, search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/pipeline.yml`
|
||||
2. `/etc/narratio/pipeline.yml`
|
||||
|
||||
### `campaign.yml`
|
||||
|
||||
Selection rules:
|
||||
|
||||
- if `--campaign-file` is set, use that path;
|
||||
- else if `--campaign <id>` is set, use `{pipeline.campaigns.root}/{id}/campaign.yml`;
|
||||
- else use `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`.
|
||||
|
||||
### `session.yml`
|
||||
|
||||
When `--session` is omitted, local search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/session.yml`
|
||||
2. `/etc/narratio/session.yml`
|
||||
|
||||
If local session discovery fails and a `session_id` is known, Narratio attempts remote session loading from:
|
||||
|
||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||
|
||||
using configured object storage.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Stable files (`speakers_file`, `autocorrect_file`, `glossary_file`, `players_file`, `party_file`) resolve from session overrides when provided, otherwise from campaign defaults.
|
||||
- Exactly one audio mode must be configured in session input:
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
|
||||
## Minimal Working Configuration
|
||||
|
||||
`pipeline.yml`
|
||||
|
||||
```yaml
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
```
|
||||
|
||||
`campaign.yml`
|
||||
|
||||
```yaml
|
||||
campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
```
|
||||
|
||||
`session.yml` (local audio)
|
||||
|
||||
```yaml
|
||||
session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
```
|
||||
|
||||
## Secrets Handling
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||
- Optionally load env files from `pipeline.secrets.env_dir`.
|
||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||
|
||||
## Publish Configuration Summary
|
||||
|
||||
Publish rules live under `pipeline.publish`.
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: manual post-publish edits
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `outputs[].source` is required.
|
||||
- `outputs[].dest` may be omitted when derivable from source.
|
||||
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
|
||||
## Full Schema
|
||||
|
||||
### Pipeline
|
||||
|
||||
| Field | Type | Required | Default / Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | empty |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||
| `pipeline.publish.upload_run` | bool | No | `true` |
|
||||
| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed JSON plus final and final-trimmed Markdown outputs |
|
||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||
| `pipeline.publish.locks[]` | list | No | empty |
|
||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||
| `pipeline.whisperx.retry_delay` | duration | No | `2s` |
|
||||
| `pipeline.whisperx.concurrency` | int | No | `2` |
|
||||
| `pipeline.seriatim.binary` | string | No | `seriatim` |
|
||||
| `pipeline.seriatim.timeout` | duration | No | `10m` |
|
||||
| `pipeline.seriatim.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.seriatim.coalesce_gap` | float | No | `3.0` |
|
||||
| `pipeline.seriatim.report` | bool | No | `true` |
|
||||
| `pipeline.seriatim.env.overlap_word_run_gap` | float | No | unset |
|
||||
| `pipeline.seriatim.env.overlap_word_run_reorder_window` | float | No | unset |
|
||||
| `pipeline.seriatim.env.backchannel_max_duration` | float | No | unset |
|
||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||
| `pipeline.audita.binary` | string | No | `audita` |
|
||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||
| `pipeline.audita.base_url` | string | No | empty |
|
||||
| `pipeline.audita.model` | string | No | empty |
|
||||
| `pipeline.audita.total_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.proposal_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.validation_model` | string | No | empty |
|
||||
| `pipeline.audita.validation_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.transcript_description` | string | No | empty |
|
||||
| `pipeline.audita.config_path` | string | No | empty |
|
||||
| `pipeline.audita.output_schema` | string | No | empty |
|
||||
| `pipeline.audita.work_dir_retention` | string | No | empty |
|
||||
| `pipeline.audita.report` | bool | No | `true` |
|
||||
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
|
||||
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.normalize.report` | bool | No | `true` |
|
||||
| `pipeline.trim.enabled` | bool | No | `true` |
|
||||
| `pipeline.trim.output_path` | string | No | `transcripts/final.trimmed.json` |
|
||||
| `pipeline.trim.bounds.prompt_id` | string | No | `dnd.session_bounds` |
|
||||
| `pipeline.trim.bounds.profile_id` | string | No | empty |
|
||||
| `pipeline.trim.bounds.transcript_input_name` | string | No | `transcript` |
|
||||
| `pipeline.trim.bounds.output_path` | string | No | `artifacts/session_bounds.json` |
|
||||
| `pipeline.trim.bounds.timeout` | duration | No | `10m` |
|
||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||
| `pipeline.render.enabled` | bool | No | `true` |
|
||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||
| `pipeline.render.include_timestamps` | bool | No | `true` |
|
||||
| `pipeline.render.include_segment_ids` | bool | No | `true` |
|
||||
| `pipeline.render.include_metadata` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.binary` | string | No | `scriptorium` |
|
||||
| `pipeline.scriptorium.config_path` | string | No | empty |
|
||||
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||
| `pipeline.notification.backend` | string | No | empty |
|
||||
| `pipeline.notification.recipient` | string | No | empty |
|
||||
| `pipeline.notification.timeout` | duration | No | empty |
|
||||
|
||||
### Notarius Output Entries
|
||||
|
||||
For each `pipeline.notarius.outputs.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||
| `module_key` | string | No | exact accepted module key when set |
|
||||
|
||||
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||
contract-incompatible lane fails extraction. See the
|
||||
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||
for compatibility ownership.
|
||||
|
||||
### Scriptorium Artifact Entries
|
||||
|
||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | bool | No | `false` if omitted |
|
||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; enabled graph must be acyclic |
|
||||
| `render_debug` | bool | No | per-artifact override |
|
||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||
| `profile_id` | string | No | empty |
|
||||
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
|
||||
| `timeout` | duration | No | artifact override |
|
||||
| `inputs` | map | No | input key names must be non-empty |
|
||||
| `vars` | map | No | values must be string or bool; `session_id` is reserved and overwritten by Narratio |
|
||||
|
||||
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
|
||||
|
||||
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `artifact` | string | No | optional passthrough adapter field |
|
||||
| `path` | string | No | optional passthrough adapter field |
|
||||
| `required` | bool | No | optional input requirement |
|
||||
|
||||
### Campaign
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `campaign_id` | string | Yes | canonical campaign identity |
|
||||
| `session_template_file` | string | No | used by `session init` when set |
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||
| `inputs.players_file` | string | Yes | stable input default |
|
||||
| `inputs.party_file` | string | Yes | stable input default |
|
||||
|
||||
### Session
|
||||
|
||||
| Field | Type | Required in session file | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `session_id` | string | Yes | must match CLI session target when provided |
|
||||
| `previous_session_id` | string | No | must not equal `session_id` |
|
||||
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted |
|
||||
| `date` | string | No | metadata |
|
||||
| `title` | string | No | metadata |
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.players_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.party_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
|
||||
|
||||
Audio rules:
|
||||
|
||||
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
See the [maintained examples index](../examples/README.md) for complete pipeline,
|
||||
campaign, session, template, and input fixtures. Keep complete copyable files
|
||||
there rather than duplicating them in this reference.
|
||||
43
docs/development.md
Normal file
43
docs/development.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Development
|
||||
|
||||
This is the first-read landing page for people and LLM coding agents working on
|
||||
Narratio. It provides a concise repository orientation and routes each kind of
|
||||
change to its canonical documentation.
|
||||
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts and generated artifacts. Start with the
|
||||
[README](../README.md) for product context,
|
||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
| --- | --- | --- |
|
||||
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internal documents. |
|
||||
| Application shape, boundaries, dependency direction, runtime invariants, safety properties, or dependencies | [Architecture](policy/architecture.md) | It defines the intended system shape, ownership, and non-goals. |
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical owners, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Adding, changing, reviewing, rewriting, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and test lifecycle decisions. |
|
||||
| CLI composition or command behavior | [Internal Overview](internal/overview.md) and [CLI Reference](cli.md) | The overview routes to command ownership; the reference owns public syntax and invocation behavior. |
|
||||
| Configuration loading, resolution, or user-visible configuration | [Internal Overview](internal/overview.md) and [Configuration](config.md) | The overview routes to implementation ownership; the reference owns fields, defaults, discovery, and validation. |
|
||||
| Session workflow, status, restore, cleanup, or object storage | [Restore Internals](internal/command-restore.md), [Workspace Internals](internal/workspace.md), [Storage Internals](internal/storage.md), [Operations](operations.md), and [Troubleshooting](troubleshooting.md) | These separate implementation mechanics, operator procedures, and symptom-driven recovery. |
|
||||
| Pipeline sequencing or the behavior of a stage | [Internal Overview](internal/overview.md) and its focused stage documents | The overview owns the implemented stage inventory and routes to each stage contract. |
|
||||
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
|
||||
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
|
||||
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
|
||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
For an existing subsystem, also inspect its focused tests and package-level
|
||||
contracts before changing behavior.
|
||||
|
||||
## Validation
|
||||
|
||||
Use focused package tests while iterating. Run the repository-wide checks when a
|
||||
change affects shared contracts, application behavior, or maintained
|
||||
documentation examples:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./cmd/narratio
|
||||
```
|
||||
35
docs/integrations/README.md
Normal file
35
docs/integrations/README.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Integrations Index
|
||||
|
||||
## Audience
|
||||
|
||||
Operators, developers, and coding agents who need to understand Narratio's
|
||||
externally observable integration boundaries.
|
||||
|
||||
## Scope
|
||||
|
||||
`docs/integrations/` is the canonical reference for protocols, invocation and
|
||||
data contracts, logical outputs, and compatibility behavior at external tool
|
||||
boundaries.
|
||||
|
||||
These documents describe what Narratio sends or invokes, what it accepts in
|
||||
return, and how failures are surfaced. Internal composition and stage mechanics
|
||||
belong in [the adapter implementation guide](../internal/adapters.md) and the
|
||||
focused stage documents.
|
||||
|
||||
## Integration Contracts
|
||||
|
||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||
discovery (`notarius run`).
|
||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||
(`scriptorium run|render`).
|
||||
- [WhisperX](./whisperx.md): speaker-audio transcription over HTTP.
|
||||
|
||||
## Related Canonical Docs
|
||||
|
||||
- [Configuration](../config.md): operator-facing configuration reference.
|
||||
- [Adapter implementation](../internal/adapters.md): shared adapter boundary and
|
||||
runner wiring.
|
||||
- [Internal documentation](../internal/overview.md): stage-specific integration
|
||||
usage and component ownership.
|
||||
@@ -1,96 +1,59 @@
|
||||
# Audita Subprocess Operations
|
||||
# Integration: Audita
|
||||
|
||||
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
||||
## Purpose
|
||||
Define the Audita adapter contract used by the `polish` stage.
|
||||
|
||||
## Recommended command form
|
||||
## External Boundary
|
||||
|
||||
Use explicit file outputs for orchestrated runs:
|
||||
Narratio invokes `audita process` as a subprocess for each polish operation.
|
||||
The configured timeout and parent cancellation bound the invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
## Request Contract
|
||||
`PolishRequest` carries:
|
||||
- required transcript/glossary/output/work-dir paths;
|
||||
- optional report path (required when report mode is enabled);
|
||||
- generated config and stdout/stderr log paths;
|
||||
- optional module/model/base-url/config/output-schema/concurrency settings.
|
||||
|
||||
Additional flags that may be situationally appropriate:
|
||||
- `--config <path>` to select an explicit versioned config file.
|
||||
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
||||
- `--work-dir <dir>` to control diagnostics location.
|
||||
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
||||
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
||||
- `--modules ...` only when intentionally overriding the default sequence.
|
||||
## Result Contract
|
||||
`PolishResult` returns:
|
||||
- processed transcript path;
|
||||
- optional report path;
|
||||
- work dir and generated-config/log paths;
|
||||
- exit code, duration, binary provenance;
|
||||
- adapter metadata map.
|
||||
|
||||
For config-driven orchestration, validate config files in CI/preflight:
|
||||
## Validation and Failure Semantics
|
||||
Construction fails for invalid static config values, including:
|
||||
- empty binary;
|
||||
- non-positive timeout;
|
||||
- invalid base URL;
|
||||
- invalid output schema;
|
||||
- invalid work-dir retention value;
|
||||
- invalid concurrency values.
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
```
|
||||
Run fails for:
|
||||
- missing required request paths;
|
||||
- missing required credential env var when configured (`llm_api_key_env`);
|
||||
- subprocess execution failure;
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
|
||||
## Stdout behavior
|
||||
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- Report JSON is never written to stdout.
|
||||
## Deterministic Behavior
|
||||
- CLI args are built from runner config + request in a fixed order.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
## Stderr behavior
|
||||
## Configuration
|
||||
|
||||
- Success path should be quiet or minimal human-readable logs.
|
||||
- Failure path writes concise human-readable errors.
|
||||
- When a diagnostics run directory exists, failure stderr includes its path.
|
||||
- Prompt/response diagnostic payloads are not streamed to stderr.
|
||||
Operator-selected values are defined under `pipeline.audita.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
## Output file behavior
|
||||
Maintained example with Audita config:
|
||||
|
||||
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
||||
- Output write failures return nonzero and surface actionable errors.
|
||||
- The command does not silently ignore output write errors.
|
||||
|
||||
## Report JSON behavior
|
||||
|
||||
- `--report-json` writes a machine-readable process report to the requested path.
|
||||
- Run-directory `report.json` is written independently under diagnostics.
|
||||
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
||||
- Report write failures return nonzero with clear stderr messaging.
|
||||
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
|
||||
- Each run creates (when possible) a per-run diagnostics directory.
|
||||
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
||||
- Failed runs retain diagnostics.
|
||||
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
||||
|
||||
## Exit codes
|
||||
|
||||
- `0`: success.
|
||||
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
||||
|
||||
Treat any nonzero as a failed subprocess invocation.
|
||||
|
||||
## Timeout and cancellation
|
||||
|
||||
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
||||
- On cancellation or timeout, the process exits nonzero and should not hang.
|
||||
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
||||
|
||||
## Secret redaction expectations
|
||||
|
||||
API keys and configured secret values are redacted from:
|
||||
- reports (`--report-json` and run-dir `report.json`);
|
||||
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
||||
- surfaced adapter/runtime errors;
|
||||
- test fixtures and regression outputs.
|
||||
|
||||
Parent-process logs should still avoid printing raw environment variables.
|
||||
|
||||
## Parent-process pipe guidance
|
||||
|
||||
To avoid deadlocks in orchestrators:
|
||||
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
||||
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
||||
- treat stderr as human-readable diagnostics, not structured data;
|
||||
- parse structured results from output/report files.
|
||||
|
||||
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
83
docs/integrations/notarius.md
Normal file
83
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Notarius Integration Contract
|
||||
|
||||
## Boundary
|
||||
|
||||
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||
|
||||
Canonical Notarius references:
|
||||
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||
|
||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||
Notarius documents as canonical when changing those values; Narratio does not
|
||||
duplicate the complete schemas.
|
||||
|
||||
## Invocation
|
||||
|
||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||
configuration path, input path, output directory, and working directory to
|
||||
absolute paths and invokes:
|
||||
|
||||
```text
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||
```
|
||||
|
||||
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||
separately as diagnostic output. Narratio applies the configured timeout and
|
||||
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||
It does not pass a Narratio session ID or run `notarius config validate`
|
||||
automatically; the configured working directory and inherited environment
|
||||
apply to the subprocess.
|
||||
|
||||
## Accepted Result
|
||||
|
||||
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||
must identify the configured pipeline, and its `index_file` must be exactly
|
||||
`index.json` beneath the reported bundle root. The production index must name
|
||||
the management files exactly as `manifest.json`, `rejected.json`, and
|
||||
`warnings.json`. All receipt, index, and lane paths must stay inside that
|
||||
bundle; symlinks and non-regular lane payloads are rejected.
|
||||
|
||||
Supported receipt and index shapes tolerate unknown fields for forward
|
||||
compatibility, while required identity, validation, count, manifest,
|
||||
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||
chunk-map and evidence-context descriptors must carry their complete generic
|
||||
contract metadata when present.
|
||||
|
||||
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||
selectable Narratio sources.
|
||||
|
||||
Each accepted configured lane is registered as
|
||||
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||
|
||||
## Failure And Compatibility Behavior
|
||||
|
||||
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||
staged bundle is promoted to durable storage.
|
||||
- Contract and external provenance metadata are preserved on lane artifact
|
||||
records and through explicit publication.
|
||||
|
||||
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||
reason-code fields for diagnostics without exposing free-form external messages
|
||||
or reading lane payload bodies.
|
||||
|
||||
Configuration fields and defaults are in [Configuration](../config.md).
|
||||
Operator paths, rerun procedures, and bundle retention are in
|
||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||
for failure recovery.
|
||||
@@ -1,339 +1,68 @@
|
||||
# Narratio -> Scriptorium CLI Integration
|
||||
# Integration: Scriptorium
|
||||
|
||||
## 1. Purpose
|
||||
## Purpose
|
||||
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
|
||||
|
||||
This document defines how Narratio should invoke Scriptorium through the **public CLI**.
|
||||
## External Boundary
|
||||
|
||||
This is a **subprocess integration contract**, not an internal Go API contract.
|
||||
|
||||
## 2. Assumptions
|
||||
|
||||
- `scriptorium` is installed and available on `PATH`.
|
||||
- Scriptorium is configured with `config.yml`.
|
||||
- `config.yml` provides `prompt_dir`, `profile_dir`, and `schema_dir` as needed.
|
||||
- Prompt and profile libraries are already deployed for the environment.
|
||||
- Narratio provides prepared artifact files (for example polished transcript, glossary, previous recap, campaign notes).
|
||||
- Initial integration is synchronous subprocess execution.
|
||||
- Narratio remains the orchestrator.
|
||||
|
||||
In normal operation, Narratio does not need to pass `--prompt-dir` and `--profile-dir` if they are supplied by Scriptorium config.
|
||||
|
||||
Narratio may pass `--config <PATH>` when it must use a non-default Scriptorium config file.
|
||||
|
||||
## 3. Core Commands Narratio May Call
|
||||
|
||||
Primary commands for subprocess integration:
|
||||
Narratio invokes Scriptorium as a subprocess in these modes:
|
||||
|
||||
- `scriptorium run`
|
||||
- `scriptorium render`
|
||||
|
||||
For production generation, use `scriptorium run`.
|
||||
|
||||
`scriptorium render` is for debugging, dry-runs, test assertions, and validating command construction without LLM execution.
|
||||
|
||||
Note: `scriptorium serve` and HTTP API exist, but they are not the initial integration path.
|
||||
|
||||
## 4. Command Selection Guidance
|
||||
|
||||
- Use `run` to generate an output artifact.
|
||||
- Use `render` to inspect the prepared prompt and effective settings without calling the LLM.
|
||||
- Use `render --format json` when Narratio/tests need structured prepare output.
|
||||
|
||||
## 5. Recommended `run` Invocation Shape
|
||||
|
||||
Production shape:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--out <output-artifact-path>
|
||||
```
|
||||
|
||||
Common optional additions:
|
||||
|
||||
- `--config <path>`: use a specific Scriptorium config file.
|
||||
- `--profile <profile_id>`: override prompt default profile.
|
||||
- `--var name=value` (repeatable): small metadata values.
|
||||
- `--input name=path` (repeatable): additional named artifacts.
|
||||
- `--timeout <duration>`: per-run timeout override.
|
||||
- Runtime model override flags (`--llm-base-url`, `--model`, etc.) only for exceptional/operator-directed cases.
|
||||
|
||||
## 6. Recommended `render` Invocation Shape
|
||||
|
||||
Human-readable debug shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format text
|
||||
```
|
||||
|
||||
Structured debug/test shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format json \
|
||||
--out <render-debug-path>
|
||||
```
|
||||
|
||||
`render` does **not** call the LLM, does **not** validate model output, and does **not** perform repair.
|
||||
|
||||
## 7. Inputs
|
||||
|
||||
- Pass inputs as repeated `--input name=path` flags.
|
||||
- `name` must match the Prompt Definition input name.
|
||||
- Prefer absolute paths, or paths relative to a working directory controlled by Narratio.
|
||||
- Pass Audita output as the primary transcript input.
|
||||
- Additional inputs may include glossary, previous recap, campaign notes, event logs, final state maps, or other prompt-specific artifacts.
|
||||
- Scriptorium reads input files directly; Narratio does not need to inline file content for CLI use.
|
||||
|
||||
## 8. Variables
|
||||
|
||||
Use repeated `--var name=value` for small metadata values.
|
||||
|
||||
Typical examples:
|
||||
|
||||
- `session_date`
|
||||
- `session_id`
|
||||
- `campaign_name`
|
||||
- `previous_session_id`
|
||||
- `output_kind`
|
||||
|
||||
Large content belongs in input files, not `--var` values.
|
||||
|
||||
## 9. Prompt IDs and Output Artifact Types
|
||||
|
||||
Narratio should treat prompt IDs as configuration, not hardcoded business logic.
|
||||
|
||||
Narratio config may map stage/output names to prompt IDs, for example:
|
||||
|
||||
- session recap prompt
|
||||
- structured event extraction prompt
|
||||
- glossary suggestion prompt
|
||||
- player-facing summary prompt
|
||||
|
||||
Prompt IDs used by Narratio should come from the deployed Scriptorium prompt library.
|
||||
|
||||
## 10. Profiles
|
||||
|
||||
- Prompts may declare `default_profile`.
|
||||
- Narratio may omit `--profile` to use prompt default profile.
|
||||
- Narratio may pass `--profile` to force profile selection.
|
||||
- This enables environment/profile selection like `local-fast`, `local-quality`, `frontier`, `batch`, or test profiles.
|
||||
- Profile names should generally be Narratio configuration values.
|
||||
|
||||
## 11. Runtime Overrides
|
||||
|
||||
Supported runtime override flags:
|
||||
|
||||
- `--llm-base-url`
|
||||
- `--model`
|
||||
- `--api-key-env`
|
||||
- `--temperature`
|
||||
- `--max-tokens`
|
||||
- `--top-p`
|
||||
- `--timeout`
|
||||
|
||||
Guidance:
|
||||
|
||||
- Keep normal model/runtime settings in Execution Profiles.
|
||||
- Use runtime overrides only for explicit per-run exceptions, tests, or operator overrides.
|
||||
- Never pass raw API keys on the command line.
|
||||
- `--api-key-env` names an environment variable; Narratio must ensure that variable is set in subprocess environment.
|
||||
|
||||
## 12. Config Behavior
|
||||
|
||||
- Default config path: `/etc/scriptorium/config.yml`.
|
||||
- `--config <PATH>` overrides default path.
|
||||
- Missing default config is allowed by Scriptorium.
|
||||
- If `--config` is provided explicitly, the file must exist and be valid.
|
||||
- CLI flags override `config.yml`.
|
||||
- `config.yml` overrides built-in application defaults.
|
||||
|
||||
Narratio can either:
|
||||
|
||||
- rely on system default config path, or
|
||||
- carry an explicit config path and pass `--config`.
|
||||
|
||||
## 13. Environment Handling
|
||||
|
||||
Subprocess environment recommendations:
|
||||
|
||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
||||
- Do not pass raw API keys as CLI arguments.
|
||||
- Avoid logging full environment dumps.
|
||||
- Capture stdout and stderr separately.
|
||||
- Use a controlled working directory.
|
||||
- Prefer absolute artifact paths.
|
||||
|
||||
## 14. Output Handling
|
||||
|
||||
For `scriptorium run`:
|
||||
|
||||
- Use `--out` when Narratio needs durable artifact files.
|
||||
- Without `--out`, artifact content is written to stdout.
|
||||
- Preferred orchestration pattern: always use `--out`, then treat the file as stage output artifact.
|
||||
- Capture stderr for diagnostics.
|
||||
|
||||
For `scriptorium render`:
|
||||
|
||||
- Use `--out` to store render diagnostics.
|
||||
- Use `--format json` when tests need to inspect selected profile, effective runtime settings, input hashes, prompt hash, and rendered messages.
|
||||
|
||||
## 15. Exit Status and Errors
|
||||
|
||||
Current CLI behavior (verified from implementation/tests):
|
||||
|
||||
- `0`: success.
|
||||
- `1`: runtime/parse/config/load/render/generation/IO error.
|
||||
- `2`: run completed but output validation failed (`ValidationFailed`).
|
||||
|
||||
Additional details:
|
||||
|
||||
- On `run`, output artifact write happens before exit code selection. If validation fails, artifact may still be written and exit code is `2`.
|
||||
- `stderr` carries both errors and normal run summary output; non-empty stderr alone does not imply failure.
|
||||
- `render` returns `0` on success and `1` on failures.
|
||||
|
||||
Narratio should treat non-zero exit codes as failed stage execution, but may record generated artifact paths if a run exited `2` and output file exists.
|
||||
|
||||
## 16. Recommended Narratio Integration Pattern
|
||||
|
||||
1. Build CLI args from Narratio stage configuration.
|
||||
2. Use subprocess context cancellation/timeout.
|
||||
3. Pass absolute input paths.
|
||||
4. Pass `--out` to a session-scoped artifact path.
|
||||
5. Add `--var` metadata values.
|
||||
6. Optionally add `--config`.
|
||||
7. Optionally add `--profile`.
|
||||
8. Ensure required API-key env vars are present.
|
||||
9. Run subprocess synchronously.
|
||||
10. Capture stdout/stderr separately.
|
||||
11. On success, store output artifact path and invocation metadata in stage artifacts.
|
||||
12. On failure, store exit code and stderr diagnostics in stage status.
|
||||
|
||||
## 17. Suggested Narratio Configuration Shape
|
||||
|
||||
Illustrative `pipeline.yml` shape:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-quality # optional
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
render_debug: false # optional artifact override
|
||||
inputs:
|
||||
transcript:
|
||||
source: trimmed_transcript
|
||||
required: true
|
||||
previous_recap:
|
||||
source: previous_session_artifact
|
||||
artifact: session_recap
|
||||
path: "" # optional
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
```
|
||||
|
||||
The key idea: map Narratio artifact names to prompt ID, optional profile, expected inputs, vars, and output destination.
|
||||
|
||||
## 18. Testing Strategy for Narratio Integration
|
||||
|
||||
- Use `scriptorium render --format json` to verify command construction without LLM calls.
|
||||
- Use dedicated test prompt/profile libraries for integration tests.
|
||||
- Use small fixture transcripts.
|
||||
- Verify missing-input failure behavior.
|
||||
- Verify prompt `default_profile` behavior.
|
||||
- Verify explicit `--profile` override behavior.
|
||||
- Verify `--config` behavior (default and explicit).
|
||||
- Verify output file creation when `--out` is used.
|
||||
- Verify stderr capture on failures.
|
||||
- Avoid real API keys in tests.
|
||||
|
||||
## 19. Security and Privacy Notes
|
||||
|
||||
- Never pass raw API keys on command line.
|
||||
- Do not log full rendered prompts by default; transcripts may contain sensitive content.
|
||||
- Avoid logging prompt content unless explicit debug mode is enabled.
|
||||
- Treat generated artifacts as potentially sensitive.
|
||||
- Use session-scoped, access-controlled output paths.
|
||||
- `api_key_env` names should come from environment management, not embedded secrets.
|
||||
|
||||
## 20. Initial D&D Artifact Generation Examples
|
||||
|
||||
These are examples only. Use prompt IDs from the deployed prompt library.
|
||||
|
||||
Session recap:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.session_recap \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input glossary=/work/session-42/glossary.yml \
|
||||
--out /work/session-42/artifacts/session_recap.md
|
||||
```
|
||||
|
||||
Structured events:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.structured_events \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--out /work/session-42/artifacts/structured_events.json
|
||||
```
|
||||
|
||||
Glossary suggestions:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.glossary_suggestions \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input previous_recap=/work/session-41/artifacts/session_recap.md \
|
||||
--out /work/session-42/artifacts/glossary_suggestions.md
|
||||
```
|
||||
|
||||
Player-facing summary:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.player_summary \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input structured_events=/work/session-42/artifacts/structured_events.json \
|
||||
--out /work/session-42/artifacts/player_summary.md
|
||||
```
|
||||
|
||||
## 21. Non-Goals
|
||||
|
||||
Initial Narratio integration should not:
|
||||
|
||||
- call Scriptorium internal Go packages
|
||||
- use HTTP API as the primary path
|
||||
- expect Scriptorium to read S3 refs directly
|
||||
- make Scriptorium responsible for Narratio stage state
|
||||
- make Scriptorium responsible for notification
|
||||
- require Scriptorium to understand D&D workflow semantics beyond prompt definitions
|
||||
|
||||
## 22. Future Extension Notes
|
||||
|
||||
Possible later extensions:
|
||||
|
||||
- HTTP API integration
|
||||
- S3 artifact references if Scriptorium adds S3 reader support
|
||||
- richer render diagnostics and policy controls
|
||||
- token budgeting/prompt-size checks
|
||||
- batch execution if Scriptorium later adds batch support
|
||||
The request timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
## Request Contract
|
||||
Both request types carry:
|
||||
- binary/config/prompt/profile IDs;
|
||||
- input map and vars map;
|
||||
- output path;
|
||||
- timeout;
|
||||
- generated config + stdout/stderr log paths;
|
||||
- optional API-key env var name;
|
||||
- optional working directory.
|
||||
|
||||
## Result Contract
|
||||
`ArtifactResult` returns:
|
||||
- output/log/generated-config paths;
|
||||
- exit code and duration;
|
||||
- command mode (`run` or `render`);
|
||||
- prompt/profile provenance;
|
||||
- `ValidationFailed` marker;
|
||||
- metadata map.
|
||||
|
||||
## Validation and Failure Semantics
|
||||
Request validation fails for:
|
||||
- missing binary, prompt id, or output path;
|
||||
- non-positive timeout;
|
||||
- empty input/var names;
|
||||
- empty input path values;
|
||||
- missing required credential env var when `APIKeyEnv` is set.
|
||||
|
||||
Run behavior:
|
||||
- subprocess errors propagate with context;
|
||||
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||
- successful subprocess still fails if output file is missing or empty.
|
||||
|
||||
Render behavior:
|
||||
- subprocess errors propagate;
|
||||
- output file must exist and be non-empty.
|
||||
|
||||
## Deterministic Behavior
|
||||
- input and var maps are sorted into deterministic `--input` and `--var` CLI args.
|
||||
- stage wiring adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream routing, overriding any configured `vars.session_id`.
|
||||
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
|
||||
- adapter is stateless and does not own artifact-selection policy.
|
||||
|
||||
## Configuration
|
||||
|
||||
Operator-selected values are defined under `pipeline.scriptorium.*`, including
|
||||
per-artifact settings under `pipeline.scriptorium.artifacts.*`, in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Maintained examples with Scriptorium config:
|
||||
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
@@ -1,403 +1,60 @@
|
||||
# seriatim
|
||||
# Integration: Seriatim
|
||||
|
||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
||||
## Purpose
|
||||
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
|
||||
|
||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
||||
## External Boundary
|
||||
|
||||
## Usage
|
||||
Narratio invokes Seriatim as a subprocess in these modes:
|
||||
|
||||
Run from source:
|
||||
- `seriatim merge`
|
||||
- `seriatim normalize`
|
||||
- `seriatim trim`
|
||||
- `seriatim render`
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file samples/raw/2026-04-19-Eric_Rakestraw.json \
|
||||
--input-file samples/raw/2026-04-19-Mike_Brown.json \
|
||||
--output-file merged.json
|
||||
```
|
||||
The configured timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
Optional report output:
|
||||
## Request/Result Contracts
|
||||
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
|
||||
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
|
||||
- `TrimRequest`/`TrimResult`: transcript trimming with required keep selector.
|
||||
- `RenderRequest`/`RenderResult`: transcript-to-markdown rendering with explicit format and render booleans.
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file eric.json \
|
||||
--input-file mike.json \
|
||||
--output-file merged.json \
|
||||
--report-file report.json
|
||||
```
|
||||
Results include output/log/config paths, timing, exit code, and metadata.
|
||||
|
||||
## CLI
|
||||
## Validation and Failure Semantics
|
||||
Runner construction validates:
|
||||
- binary presence;
|
||||
- timeout > 0;
|
||||
- supported output schema (`seriatim-minimal|seriatim-intermediate|seriatim-full`);
|
||||
- non-negative coalesce gap.
|
||||
|
||||
```text
|
||||
seriatim merge [flags]
|
||||
```
|
||||
Invocation fails on:
|
||||
- missing required request paths/inputs;
|
||||
- invalid normalize schema override;
|
||||
- unsupported render format;
|
||||
- subprocess failure;
|
||||
- invalid JSON outputs for merge/normalize/trim;
|
||||
- missing `segments` array for normalize/trim transcript outputs;
|
||||
- empty render output files.
|
||||
|
||||
Global flags:
|
||||
When report paths are provided/enabled, report files must parse as JSON.
|
||||
|
||||
| Flag | Description |
|
||||
| --- | --- |
|
||||
| `--help` | Show command help. |
|
||||
| `--version` | Show application version. Local builds default to `dev`; release builds inject the release version. |
|
||||
## Deterministic Behavior
|
||||
- argument ordering is deterministic per command construction.
|
||||
- merge env overrides are explicit (`SERIATIM_*`) and only emitted when configured.
|
||||
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
|
||||
- adapter does not write manifests or choose stage inputs.
|
||||
|
||||
`merge` flags:
|
||||
## Configuration
|
||||
|
||||
| Flag | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `--input-file` | Yes | none | Input transcript JSON file. Repeat once per speaker/input file. |
|
||||
| `--output-file` | Yes | none | Merged transcript JSON output path. |
|
||||
| `--report-file` | No | none | Optional report JSON output path. |
|
||||
| `--speakers` | No | none | Speaker map YAML file. When omitted, input file basenames are used as speaker labels. |
|
||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
||||
| `--output-schema` | No | `seriatim-intermediate` | JSON output contract. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. If omitted, the runtime default is used; consumers that depend on a specific shape should set this explicitly. |
|
||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
||||
Operator-selected values are defined under `pipeline.seriatim.*` and
|
||||
`pipeline.render.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Environment variables:
|
||||
Maintained examples with Seriatim config:
|
||||
|
||||
| Environment Variable | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | Output schema used when `--output-schema` is not explicitly provided. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. The CLI flag takes precedence. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
||||
|
||||
## Input JSON Format
|
||||
|
||||
Each input file must be valid JSON with a top-level `segments` array. The current parser accepts the WhisperX segment subset needed for merging:
|
||||
|
||||
```json
|
||||
{
|
||||
"segments": [
|
||||
{
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"words": [
|
||||
{"word": "Hello", "start": 1.25, "end": 1.55, "score": 0.98},
|
||||
{"word": "there.", "start": 1.7, "end": 2.0}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Required segment fields:
|
||||
|
||||
- `start`: number, must be `>= 0`.
|
||||
- `end`: number, must be `>= start`.
|
||||
- `text`: string.
|
||||
|
||||
Optional word fields:
|
||||
|
||||
- `words`: array of word timing objects.
|
||||
- `words[].word`: string.
|
||||
- `words[].start`: optional number, must be `>= 0` when present.
|
||||
- `words[].end`: optional number, must be `>= start` when present with `start`.
|
||||
- `words[].score`: optional number.
|
||||
- `words[].speaker`: optional raw speaker label string.
|
||||
|
||||
Word-level timing is preserved internally for overlap resolution. If a word is missing `start` or `end`, seriatim keeps the word text, emits a warning in the optional report, and does not use that word as a timing anchor. Word timing is not emitted in the final JSON artifact.
|
||||
|
||||
## Speaker Map Format
|
||||
|
||||
`speakers.yml` maps input files to canonical speaker names using ordered substring rules:
|
||||
|
||||
This file is optional. If `--speakers` is omitted, `seriatim` uses each input file basename as the segment speaker label.
|
||||
|
||||
```yaml
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
|
||||
- speaker: "Mike Brown"
|
||||
match:
|
||||
- "Mike_Brown"
|
||||
- "mb"
|
||||
```
|
||||
|
||||
For each `--input-file`, `seriatim` takes the file basename and evaluates the rules in order. The first rule with a matching substring wins, and no later rules are evaluated.
|
||||
|
||||
For example, this input:
|
||||
|
||||
```text
|
||||
samples/raw/2026-04-19-Eric_Rakestraw.json
|
||||
```
|
||||
|
||||
matches this rule because the basename contains `Eric_Rakestraw`:
|
||||
|
||||
```yaml
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
```
|
||||
|
||||
Important details:
|
||||
|
||||
- Matching is against the input file basename, not the full path.
|
||||
- Matching is case-insensitive.
|
||||
- Rules are evaluated from first to last.
|
||||
- Each rule must have a non-empty `speaker`.
|
||||
- Each rule must have at least one non-empty `match` string.
|
||||
- Duplicate speaker names are invalid.
|
||||
- Every input file must match at least one rule or the command fails.
|
||||
|
||||
Deprecated old format:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
eric.json:
|
||||
speaker: "Eric Rakestraw"
|
||||
```
|
||||
|
||||
The old `inputs:` direct mapping format is no longer supported.
|
||||
|
||||
## Output JSON Format
|
||||
|
||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
||||
|
||||
The named schemas are stable public contracts. If a consumer depends on a specific shape, it should request that schema explicitly at runtime. The runtime default selection may change in a future release.
|
||||
|
||||
The `seriatim-intermediate` schema is the current default selection when neither `--output-schema` nor `SERIATIM_OUTPUT_SCHEMA` is set. It stays close to the minimal schema, but adds optional `categories` on each segment:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-intermediate"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there.",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-full` schema uses the full seriatim envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"input_reader": "json-files",
|
||||
"input_files": ["eric.json", "mike.json"],
|
||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "resolve-danglers", "coalesce", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
||||
"output_modules": ["json"]
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"source": "eric.json",
|
||||
"source_segment_index": 0,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"overlap_group_id": 1
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"source": "eric.json",
|
||||
"source_ref": "word-run:1:1:1",
|
||||
"derived_from": ["eric.json#0"],
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 2.0,
|
||||
"end": 2.5,
|
||||
"text": "Resolved word run",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
],
|
||||
"overlap_groups": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 4.0,
|
||||
"segments": ["eric.json#0", "mike.json#0"],
|
||||
"speakers": ["Eric Rakestraw", "Mike Brown"],
|
||||
"class": "unknown",
|
||||
"resolution": "unresolved"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-minimal` schema emits minimal metadata and compact ordered segments:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-minimal"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
||||
|
||||
Intermediate output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
||||
|
||||
Segments are sorted deterministically by:
|
||||
|
||||
```text
|
||||
(start, end, source, source_segment_index/source_ref, speaker)
|
||||
```
|
||||
|
||||
Final segment IDs are assigned after sorting and start at `1`.
|
||||
|
||||
The public Go output contract is available from:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
```
|
||||
|
||||
The same package embeds machine-readable JSON Schemas in `schema/full-output.schema.json`, `schema/intermediate-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
||||
|
||||
## Overlap Detection
|
||||
|
||||
The default postprocessing pipeline detects overlapping segment groups.
|
||||
|
||||
Overlap behavior:
|
||||
|
||||
- A strict timing overlap is required: `next.start < current_group_end`.
|
||||
- Segments that only touch at a boundary are not grouped.
|
||||
- Groups require at least two distinct speakers.
|
||||
- Transitive overlaps are grouped together.
|
||||
- Segments in detected groups receive `overlap_group_id`.
|
||||
- `overlap_groups[].segments` contains stable references in `source#source_segment_index` format.
|
||||
- `class` is currently `unknown`.
|
||||
- `resolution` is `unresolved` until `resolve-overlaps` replaces the group.
|
||||
|
||||
## Overlap Resolution
|
||||
|
||||
The default postprocessing pipeline runs `detect-overlaps`, then `resolve-overlaps`, then `backchannel`, then `filler`, then `resolve-danglers`, then `coalesce`, then a second `detect-overlaps` pass.
|
||||
|
||||
For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word timing to build smaller word-run replacement segments:
|
||||
|
||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
||||
- Once a segment is selected for replacement, all timed words from that segment participate in word-run construction; the window controls segment selection, not per-word clipping.
|
||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
||||
- The default word-run gap is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
||||
- The default word-run reorder window is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` to a positive number of seconds to override the default.
|
||||
- Replacement segment text is built by joining word text with single spaces.
|
||||
- Replacement segments include `source_ref` and `derived_from`.
|
||||
- Replacement segments omit `source_segment_index` because they are derived from one or more original segments.
|
||||
- Resolved overlap groups are removed before the second detection pass.
|
||||
- Replacement segments are left without `overlap_group_id` until the second detection pass annotates any remaining overlap.
|
||||
- If a speaker has no usable word timing in a group, that speaker's original segment is kept.
|
||||
- If no speakers in a group have usable word timing, the original group and annotations remain unchanged.
|
||||
|
||||
## Backchannels
|
||||
|
||||
The default pipeline runs `backchannel` before `coalesce`. It tags short acknowledgement segments with:
|
||||
|
||||
```json
|
||||
"categories": ["backchannel"]
|
||||
```
|
||||
|
||||
Backchannel matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires a matching acknowledgement phrase, no more than three whitespace-delimited words, and duration no greater than `SERIATIM_BACKCHANNEL_MAX_DURATION` seconds. The default maximum duration is `2.0` seconds.
|
||||
|
||||
## Fillers
|
||||
|
||||
The default pipeline runs `filler` after `backchannel` and before `coalesce`. It tags short filler utterances with:
|
||||
|
||||
```json
|
||||
"categories": ["filler"]
|
||||
```
|
||||
|
||||
Filler matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires only filler tokens such as `um`, `uh`, `er`, `erm`, `ah`, `eh`, `hmm`, `mm`, or repeated combinations of those tokens. Matching segments must contain no more than three whitespace-delimited words and have duration no greater than `SERIATIM_FILLER_MAX_DURATION` seconds. The default maximum duration is `1.25` seconds.
|
||||
|
||||
## Dangler Resolution
|
||||
|
||||
The default pipeline runs `resolve-danglers` before `coalesce` and before the second overlap detection pass. It repairs short derived fragments when they share provenance with a nearby segment:
|
||||
|
||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
||||
- Dangling-start fragments have no more than two words.
|
||||
- Matching uses same-speaker segments with any shared `derived_from` value.
|
||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
||||
|
||||
## Coalescing
|
||||
|
||||
The default pipeline runs `coalesce` after `resolve-danglers` and before the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
||||
|
||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
||||
|
||||
Different-speaker backchannel and filler segments do not block coalescing of surrounding same-speaker segments. Same-speaker backchannel and filler segments are merged normally when they are within `--coalesce-gap`. When same-speaker segments are coalesced, any `backchannel` or `filler` category from the merged inputs is dropped from the coalesced segment.
|
||||
|
||||
## Autocorrect
|
||||
|
||||
Autocorrect is included in the default postprocessing pipeline. If `--autocorrect` is omitted, the module leaves transcript text unchanged and records a skip event in the optional report.
|
||||
|
||||
Enable corrections by passing `--autocorrect`:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file input.json \
|
||||
--autocorrect autocorrect.yml \
|
||||
--output-file merged.json
|
||||
```
|
||||
|
||||
`autocorrect.yml` format:
|
||||
|
||||
```yaml
|
||||
autocorrect:
|
||||
- target: "Hrank"
|
||||
match:
|
||||
- "hrank"
|
||||
- "Frank"
|
||||
|
||||
- target: "Mike Brown"
|
||||
match:
|
||||
- "Mike Pat"
|
||||
```
|
||||
|
||||
Matching behavior:
|
||||
|
||||
- Matching is case-sensitive.
|
||||
- Matches apply only to whole tokens, not substrings inside larger words.
|
||||
- Punctuation and whitespace can surround a match.
|
||||
- Multi-word and hyphenated matches are supported.
|
||||
- Duplicate match strings are invalid, including duplicates across separate rules.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
- Only JSON input is supported.
|
||||
- Overlap resolution depends on WhisperX word timing; groups without usable word timing remain unresolved.
|
||||
- Alternate output formats are not implemented yet.
|
||||
|
||||
## Release Builds
|
||||
|
||||
Local builds record version metadata as `dev`. Release builds should inject the release version with `ldflags`:
|
||||
|
||||
```sh
|
||||
go build -ldflags "-X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=v1.0.0" ./cmd/seriatim
|
||||
```
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
66
docs/integrations/whisperx.md
Normal file
66
docs/integrations/whisperx.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Integration: WhisperX
|
||||
|
||||
## Purpose
|
||||
|
||||
WhisperX transcribes each prepared speaker audio file for Narratio's
|
||||
`transcribe` stage. Narratio uses an HTTP boundary and installs each successful
|
||||
response as that speaker's raw transcript JSON.
|
||||
|
||||
## HTTP Boundary
|
||||
|
||||
Narratio sends an HTTP `POST` to the configured transcription URL using
|
||||
`multipart/form-data` with:
|
||||
|
||||
- `file`: the audio file, retaining its base filename; and
|
||||
- `language`: the configured language string.
|
||||
|
||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||
not currently require a more specific response schema at this boundary.
|
||||
|
||||
## Request And Result Contract
|
||||
|
||||
Each adapter request identifies a speaker, a readable audio file, and the
|
||||
destination for the raw transcript. The HTTP request carries the audio and
|
||||
language; the speaker identifier remains Narratio orchestration metadata.
|
||||
|
||||
On success, Narratio atomically writes the response body to the requested
|
||||
destination. The adapter result reports that logical output together with the
|
||||
attempt count, final HTTP status when available, elapsed duration, and adapter
|
||||
identity metadata. A failed or invalid response is not installed as the
|
||||
transcript output.
|
||||
|
||||
## Retry, Timeout, And Cancellation
|
||||
|
||||
- The configured timeout applies independently to each HTTP attempt.
|
||||
- `retries` means additional attempts after the first.
|
||||
- HTTP `429`, HTTP `5xx`, attempt timeouts, and network errors are retryable.
|
||||
- Other HTTP `4xx` responses and explicit cancellation are not retryable.
|
||||
- Narratio waits the configured retry delay between attempts and aborts that
|
||||
wait when the parent context is canceled.
|
||||
|
||||
## Validation And Failure Semantics
|
||||
|
||||
Client construction rejects a missing or invalid absolute transcription URL,
|
||||
a missing language, a non-positive timeout, negative retries, or a negative
|
||||
retry delay. A request fails before transmission when its audio or output path
|
||||
is missing.
|
||||
|
||||
Non-`2xx` status, transport failure, response-size overflow, invalid JSON, or
|
||||
failure to install the output causes the transcription to fail. Errors include
|
||||
attempt context, and the result retains attempts, final status when available,
|
||||
and elapsed duration for diagnostics.
|
||||
|
||||
## Determinism And Concurrency
|
||||
|
||||
Each audio request has stable multipart field names, and successful bytes are
|
||||
installed atomically. The transcribe stage may process speaker files in
|
||||
parallel, bounded by the configured concurrency. It records results in stable
|
||||
speaker order after all work completes; any speaker failure fails the stage.
|
||||
|
||||
## Related Canonical Docs
|
||||
|
||||
- [Configuration](../config.md#pipeline) defines the operator-selected
|
||||
WhisperX URL, language, timeouts, retry policy, and concurrency.
|
||||
- [Adapter implementation](../internal/adapters.md) describes internal wiring.
|
||||
- [Transcribe stage](../internal/stage-transcribe.md) describes stage mechanics,
|
||||
durable artifacts, and manifests.
|
||||
78
docs/internal/adapters.md
Normal file
78
docs/internal/adapters.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# Internal: Adapters
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the adapter interfaces and production composition used by application
|
||||
and stage orchestration. Externally observable protocols and formats belong in
|
||||
the [integration contracts](../integrations/).
|
||||
|
||||
## Adapter Boundaries
|
||||
|
||||
Narratio stage logic depends on adapter interfaces, not transport-specific details.
|
||||
|
||||
Primary adapters:
|
||||
|
||||
- `whisperx.Client`
|
||||
- `seriatim.Runner`
|
||||
- `audita.Runner`
|
||||
- `scriptorium.Runner`
|
||||
- `notarius.Runner`
|
||||
- `storage.ObjectStore`
|
||||
- `notify.Sender`
|
||||
|
||||
## Ownership
|
||||
|
||||
Adapters own:
|
||||
|
||||
- HTTP/subprocess/SDK argument and transport details.
|
||||
- Backend-specific request/response mapping.
|
||||
|
||||
Adapters do not own:
|
||||
|
||||
- stage ordering/skip/force logic;
|
||||
- manifest transitions;
|
||||
- canonical path policy.
|
||||
|
||||
## Default Wiring
|
||||
|
||||
`internal/app/runner.go` initializes default adapters when not injected:
|
||||
|
||||
- WhisperX HTTP client from pipeline config.
|
||||
- Seriatim subprocess runner.
|
||||
- Audita subprocess runner.
|
||||
- Scriptorium subprocess runner.
|
||||
- Notarius subprocess runner when extraction is enabled.
|
||||
- Noop notifier (`notify.NoopSender`).
|
||||
- Object store only when required by selected stages/config.
|
||||
|
||||
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||
configured filesystem secrets before adapter initialization.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
- `internal/adapters/seriatim/subprocess_test.go`
|
||||
- `internal/adapters/audita/subprocess_test.go`
|
||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||
- `internal/adapters/notarius/subprocess_test.go`
|
||||
- `internal/adapters/storage/*_test.go`
|
||||
- `internal/app/runner_test.go`
|
||||
|
||||
See the [WhisperX](../integrations/whisperx.md),
|
||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||
[Scriptorium](../integrations/scriptorium.md), and
|
||||
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||
externally visible boundary. Operator-selected values belong in
|
||||
[Configuration](../config.md).
|
||||
161
docs/internal/artifacts.md
Normal file
161
docs/internal/artifacts.md
Normal file
@@ -0,0 +1,161 @@
|
||||
# Internal: Artifacts
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the artifact registry, runtime catalog, resolver, previous-input
|
||||
requirements, and shared remote current-state mechanics implemented by
|
||||
`internal/artifacts`. Configuration fields that accept source IDs belong in
|
||||
[Configuration](../config.md); physical placement belongs in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Built-in Source IDs
|
||||
|
||||
The internal registry recognizes these stable built-in source IDs:
|
||||
|
||||
- `narratio.transcript.base`
|
||||
- `narratio.transcript.polished`
|
||||
- `narratio.transcript.final`
|
||||
- `narratio.transcript.final_trimmed`
|
||||
- `narratio.transcript.final_markdown`
|
||||
- `narratio.transcript.final_trimmed_markdown`
|
||||
- `narratio.bounds.session`
|
||||
|
||||
Registry entries bind each ID to its producer, output kind, canonical fallback,
|
||||
and content validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
|
||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
All formats are validated by strict source-policy rules. Extraction sources are
|
||||
registered only from `pipeline.notarius.outputs`; the Notarius index has no
|
||||
selectable source ID.
|
||||
|
||||
## Runtime Catalog
|
||||
|
||||
`ArtifactCatalog` tracks:
|
||||
|
||||
- `planned`: source registered for run context;
|
||||
- `executable`: selected and enabled for analyze execution;
|
||||
- `available`: local file exists and validates;
|
||||
- `provenance`: availability source.
|
||||
|
||||
Current provenance values:
|
||||
|
||||
- `generated.current_analyze_run`
|
||||
- `filesystem.disabled_artifact_output`
|
||||
- `manifest.inputs.previous_cache`
|
||||
- `current_session.previous_cache`
|
||||
|
||||
## Resolution Rules
|
||||
|
||||
Built-ins:
|
||||
|
||||
1. manifest producer outputs (when present)
|
||||
2. canonical session-path fallback
|
||||
|
||||
Configured sources (`narratio.artifact.*`):
|
||||
|
||||
- resolve only through runtime catalog availability.
|
||||
|
||||
Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
- use the shared registration and manifest hydration path in
|
||||
`extraction_catalog.go`;
|
||||
- require a current successful extract record with the exact configured source,
|
||||
compatible contract and Notarius provenance, a confined regular durable
|
||||
payload, and matching checksum; and
|
||||
- are never inferred by scanning the Notarius bundle directory.
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
- prefer manifest-backed previous-input paths;
|
||||
- fallback to existing previous-cache filesystem paths.
|
||||
|
||||
Validation by content type:
|
||||
|
||||
- transcript JSON built-ins: JSON with top-level `segments` array;
|
||||
- transcript Markdown built-ins: non-empty text file;
|
||||
- bounds built-in: valid JSON;
|
||||
- configured/previous-session artifact files: non-empty text file.
|
||||
|
||||
## Previous Requirement Collection
|
||||
|
||||
`CollectPreviousArtifactRequirements`:
|
||||
|
||||
- scans enabled configured artifacts only;
|
||||
- extracts only canonical previous-session sources;
|
||||
- deduplicates by artifact key;
|
||||
- merges required and optional references (required wins);
|
||||
- returns deterministic ordering and source locations.
|
||||
|
||||
## Current-State Helpers
|
||||
|
||||
Artifacts package owns shared remote current-state loading mechanics used by
|
||||
restore, status and validation checks, and previous-cache planning.
|
||||
|
||||
Core helpers:
|
||||
|
||||
- `LoadCurrentRunPointer`
|
||||
- `LoadCurrentManifest`
|
||||
- `LoadCurrentState`
|
||||
- `ValidateCurrentStateIdentity`
|
||||
|
||||
Typed missing-state errors:
|
||||
|
||||
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
||||
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
||||
|
||||
Identity validation supports caller-provided expectations:
|
||||
|
||||
- expected campaign;
|
||||
- expected session ID;
|
||||
- expected run ID, or pointer/manifest run-ID consistency check.
|
||||
|
||||
Caller policy is intentionally outside artifacts helpers:
|
||||
|
||||
- some callers fail on missing current state;
|
||||
- some callers downgrade missing state to status/findings;
|
||||
- some callers skip optional behavior when state is missing.
|
||||
|
||||
## Key Path Helpers
|
||||
|
||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||
|
||||
- session/work/run paths;
|
||||
- previous-cache paths;
|
||||
- spool/cache paths;
|
||||
- S3 session/run/current-state key layout.
|
||||
|
||||
See [Workspace Internals](workspace.md) for how callers consume local helpers
|
||||
and [Operations](../operations.md#local-state-layout) for the authoritative
|
||||
physical layout.
|
||||
|
||||
## Invariants
|
||||
|
||||
- source ID formats are stable contracts;
|
||||
- artifact resolution is deterministic and manifest-aware;
|
||||
- extraction sources are available only from a compatible successful manifest
|
||||
record;
|
||||
- previous-session source resolution in `analyze` is local-only;
|
||||
- remote current-state key construction remains centralized in artifacts helpers.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||
`internal/artifacts/extraction_catalog.go`
|
||||
- Current state: `internal/artifacts/current_state.go`
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||
`internal/artifacts/catalog_test.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`,
|
||||
`internal/artifacts/current_state_test.go`,
|
||||
`internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/previous_requirements_test.go`
|
||||
81
docs/internal/command-restore.md
Normal file
81
docs/internal/command-restore.md
Normal file
@@ -0,0 +1,81 @@
|
||||
# Internal: Command Restore
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the implemented restore discovery, planning, installation, and
|
||||
reporting flow in `internal/app`. User invocation belongs in
|
||||
[CLI](../cli.md#session-restore), and the operator recovery procedure and
|
||||
physical restore scope belong in
|
||||
[Operations](../operations.md#restore-workflow).
|
||||
|
||||
Restore is split into explicit phases so remote authority, local conflict
|
||||
policy, and filesystem mutation can be tested independently.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Discovery delegates current-state pointer and manifest loading to
|
||||
`internal/artifacts`, then validates the result against the resolved request:
|
||||
|
||||
- campaign must match;
|
||||
- session ID must match.
|
||||
|
||||
Restore treats any missing or invalid remote current state as a command error.
|
||||
|
||||
## Planning Contract
|
||||
|
||||
Restore planner action kinds:
|
||||
|
||||
- `download`;
|
||||
- `skip_same`;
|
||||
- `conflict`.
|
||||
|
||||
Planner behavior:
|
||||
|
||||
- remote list scope is the resolved session prefix;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted by local relative path and then remote key;
|
||||
- force converts differing local targets from conflicts to downloads.
|
||||
|
||||
Previous-cache files are planned separately through `previouscache.BuildPlan`
|
||||
when configured previous-session requirements exist.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
Execution order and safety:
|
||||
|
||||
- non-manifest downloads happen before manifest install;
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files plus atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- failed installs do not roll back files already written in the same execution.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- supports cache-hit reuse without object redownload.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
- dry-run mode prints a summary and performs no local writes;
|
||||
- execution mode persists the canonical restore report described in
|
||||
[Operations](../operations.md#restore-workflow);
|
||||
- report includes plan counts, per-action status, and execution failures.
|
||||
|
||||
## Invariants
|
||||
|
||||
- restore uses committed remote current state as authority;
|
||||
- `current/run_id.txt` is the remote publish commit marker;
|
||||
- restore does not execute pipeline stages.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Discovery: `internal/app/restore_discovery.go`
|
||||
- Planning: `internal/app/restore_plan.go`, `internal/previouscache`
|
||||
- Execution: `internal/app/restore_execute.go`
|
||||
- Reporting and command coordination: `internal/app/restore_report.go`,
|
||||
`internal/app/restore.go`
|
||||
- Tests: `internal/app/restore_discovery_test.go`,
|
||||
`internal/app/restore_plan_test.go`,
|
||||
`internal/app/restore_execution_test.go`,
|
||||
`internal/app/restore_workflow_test.go`
|
||||
100
docs/internal/manifest.md
Normal file
100
docs/internal/manifest.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Internal: Manifest
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the session-progress and invocation-audit models implemented by
|
||||
`internal/manifest`. Physical manifest placement belongs in
|
||||
[Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Session Manifest
|
||||
|
||||
`manifest.Manifest` records:
|
||||
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map
|
||||
|
||||
The model admits these stage states:
|
||||
|
||||
- `pending`
|
||||
- `running`
|
||||
- `succeeded`
|
||||
- `failed`
|
||||
- `skipped`
|
||||
- `stale`
|
||||
- `interrupted`
|
||||
|
||||
## Run Manifest
|
||||
|
||||
`manifest.RunManifest` is created for each invocation and records:
|
||||
|
||||
- invocation identity and `force` flag
|
||||
- requested stages
|
||||
- per-stage action (`run` or `skip`)
|
||||
- per-stage status
|
||||
- overall run status (`running`, `succeeded`, `failed`)
|
||||
|
||||
## Persistence Semantics
|
||||
|
||||
`manifest.LocalStore`:
|
||||
|
||||
- validates loaded documents;
|
||||
- normalizes missing maps/stage records;
|
||||
- writes atomically via temp file + rename;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
## Execution Semantics
|
||||
|
||||
The application runner marks an executing stage running and then succeeded or
|
||||
failed in both manifests, persisting each transition. On success it records
|
||||
outputs, logs, generated configuration references, and metadata. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded downstream session-stage records stale.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, and metadata. Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins. Invocation run manifests remain immutable audit
|
||||
records of their own outcomes.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in both manifests, clears older outputs for the
|
||||
session-stage record along with older logs, generated configuration references,
|
||||
and metadata, then applies any bounded details from the current skip and
|
||||
continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs.
|
||||
|
||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||
the `skip` action and reason. The session manifest deliberately retains its
|
||||
existing succeeded record because it remains the cross-invocation progress
|
||||
authority. Stages with a resume validator, currently extraction, may reject an
|
||||
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||
runner marks it stale and executes it.
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale downstream succeeded stages.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Models and transitions: `internal/manifest/manifest.go`,
|
||||
`internal/manifest/run_manifest.go`
|
||||
- Persistence and validation: `internal/manifest/store.go`
|
||||
- Package tests: `internal/manifest/*_test.go`
|
||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||
`internal/app/run_stage_test.go`
|
||||
98
docs/internal/overview.md
Normal file
98
docs/internal/overview.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# Internal Overview
|
||||
|
||||
This document is the implemented component map for Narratio. Normative system
|
||||
boundaries and dependency direction belong in
|
||||
[Architecture](../policy/architecture.md). User and operator contracts belong
|
||||
in the [CLI](../cli.md), [Configuration](../config.md),
|
||||
[Operations](../operations.md), and [Troubleshooting](../troubleshooting.md).
|
||||
Externally observable tool and format contracts belong under
|
||||
[Integrations](../integrations/).
|
||||
|
||||
## Execution Path
|
||||
|
||||
```text
|
||||
cmd/narratio -> internal/app -> configuration and production composition
|
||||
-> internal/stage -> adapters and external systems
|
||||
-> manifests and artifact resolution -> durable local/remote output
|
||||
```
|
||||
|
||||
The executable delegates process behavior to the application boundary. The
|
||||
application resolves configuration, composes concrete collaborators, acquires
|
||||
session safety controls, and runs commands. Pipeline commands execute the
|
||||
canonical stage sequence through adapter interfaces, while manifests record
|
||||
progress and artifact services resolve durable inputs and outputs.
|
||||
|
||||
## Components
|
||||
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
|
||||
| Configuration | `internal/config` | Strict YAML loading, discovery, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
|
||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, `internal/fileops` | Transcript and artifact data contracts plus narrow atomic filesystem helpers. |
|
||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||
|
||||
The application boundary composes concrete implementations. Stages depend on
|
||||
Narratio-level contracts; external transport and SDK details remain in
|
||||
adapters. The normative rules for these relationships remain in
|
||||
[Architecture](../policy/architecture.md).
|
||||
|
||||
## Pipeline Stage Set
|
||||
|
||||
The implemented canonical order is:
|
||||
|
||||
1. [`prepare`](stage-prepare.md)
|
||||
2. [`transcribe`](stage-transcribe.md)
|
||||
3. [`merge`](stage-merge.md)
|
||||
4. [`polish`](stage-polish.md)
|
||||
5. [`normalize`](stage-normalize.md)
|
||||
6. [`trim`](stage-trim.md)
|
||||
7. [`extract`](stage-extract.md)
|
||||
8. [`render`](stage-render.md)
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (placeholder)
|
||||
|
||||
`notify` currently has optional notifier call behavior and no persisted pipeline
|
||||
outputs; its default collaborator is a no-op sender. The focused stage
|
||||
documents own implementation mechanics. The
|
||||
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
|
||||
and execution semantics.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
resolution, previous requirements, and current-state helpers.
|
||||
- [Manifest Internals](manifest.md): session and run records, persistence, and
|
||||
execution transitions.
|
||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||
guardrails.
|
||||
- [Restore Internals](command-restore.md): discovery, planning, execution, and
|
||||
reporting.
|
||||
- [`prepare`](stage-prepare.md)
|
||||
- [`transcribe`](stage-transcribe.md)
|
||||
- [`merge`](stage-merge.md)
|
||||
- [`polish`](stage-polish.md)
|
||||
- [`normalize`](stage-normalize.md)
|
||||
- [`trim`](stage-trim.md)
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`render`](stage-render.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
|
||||
Use this map to find an owner, then read its focused documentation and tests
|
||||
before changing behavior.
|
||||
|
||||
The stage registry is implemented in `internal/stage/placeholders.go` and its
|
||||
ordering is protected by `internal/app/planner_test.go`. Cross-invocation skip,
|
||||
force, failure, and invalidation behavior is exercised in
|
||||
`internal/app/runner_test.go` and `internal/app/run_stage_test.go`.
|
||||
60
docs/internal/stage-analyze.md
Normal file
60
docs/internal/stage-analyze.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Stage: analyze
|
||||
|
||||
## Purpose
|
||||
|
||||
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
|
||||
|
||||
## Inputs
|
||||
|
||||
- configured artifacts from `pipeline.scriptorium.artifacts`
|
||||
- optional selected artifact keys supplied through the stage environment
|
||||
- built-in/configured/previous-session source references in artifact inputs
|
||||
|
||||
Supported source families:
|
||||
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
|
||||
`narratio.input.glossary`
|
||||
- configured artifacts: `narratio.artifact.<key>`
|
||||
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||
|
||||
## Outputs
|
||||
|
||||
- one materialized output per executed configured artifact (`output_path`)
|
||||
- stage metadata describing selected/generated/reused artifacts
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
|
||||
- builds runtime artifact catalog (built-ins + configured artifacts).
|
||||
- marks non-executable configured artifacts as reusable when output files already exist.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- resolves required/optional inputs per artifact source definition.
|
||||
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
- validates non-empty output files and materializes canonical outputs.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- required missing configured/previous-session inputs fail.
|
||||
- missing required prepared stable input source includes prepare rerun guidance.
|
||||
- missing required previous-session source includes prepare rerun guidance.
|
||||
- missing required `narratio.transcript.final_markdown` or
|
||||
`narratio.transcript.final_trimmed_markdown` inputs includes render rerun
|
||||
guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#scriptorium-artifact-entries) owns artifact
|
||||
fields and source-selection rules.
|
||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||
- Implementation and tests: `internal/stage/analyze.go`,
|
||||
`internal/stage/analyze_test.go`
|
||||
84
docs/internal/stage-extract.md
Normal file
84
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Internal: Extract Stage
|
||||
|
||||
## Responsibility
|
||||
|
||||
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||
|
||||
The external protocol is documented in the
|
||||
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||
fields belong in [Configuration](../config.md), and physical paths and force
|
||||
procedures belong in [Operations](../operations.md).
|
||||
|
||||
## Lifecycle
|
||||
|
||||
`internal/stage/extract.go`:
|
||||
|
||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||
2. resolves and fingerprints the Notarius invocation contract;
|
||||
3. creates a run-local staging directory and invokes the injected
|
||||
`notarius.Runner`;
|
||||
4. validates the successful receipt, confined index, configured required lane
|
||||
descriptors, and regular payload files;
|
||||
5. atomically promotes the complete bundle to its immutable durable location;
|
||||
6. records one non-selectable `notarius_index` output and one selectable
|
||||
`notarius_lane` output per configured lane; and
|
||||
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||
Scriptorium and publish resolution.
|
||||
|
||||
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||
Narratio run ID, and invocation fingerprint. Validation completes before
|
||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded downstream stages stale. Repeating the same disabled self-skip
|
||||
with no outputs is stable and does not repeatedly invalidate downstream stages.
|
||||
|
||||
## Resume Validation
|
||||
|
||||
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||
record succeeded and still matches the current invocation fingerprint. The
|
||||
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||
timeout, working directory, and sorted configured output contracts.
|
||||
|
||||
The validator then checks the producing run identity, canonical immutable
|
||||
bundle root, path confinement and absence of symlink components, receipt
|
||||
identity, exactly one canonical index, the exact configured source set,
|
||||
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||
return an error rather than silently accepting or replacing data.
|
||||
|
||||
The fingerprint cannot observe files imported by Notarius configuration,
|
||||
profile contents, prompt/module definitions, or other transitive inputs.
|
||||
Operators must force extraction after changing any such input.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||
promotion errors fail the stage through ordinary manifest transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and promotion.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier invocation manifest and immutable promoted bundle remain
|
||||
available for audit and recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||
`internal/stage/extract_resume.go`,
|
||||
`internal/stage/extract_test.go`
|
||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||
`internal/adapters/notarius/subprocess_test.go`
|
||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`
|
||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||
37
docs/internal/stage-merge.md
Normal file
37
docs/internal/stage-merge.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# Stage: merge
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/raw/*.json`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- optional `artifacts/seriatim.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers and validates raw transcript inputs.
|
||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||
- validates merged transcript and optional report JSON.
|
||||
- materializes canonical outputs and records stage logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- merge always consumes normalized forms of raw inputs.
|
||||
- base transcript must validate before stage success.
|
||||
- report output is config-gated.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Seriatim values.
|
||||
- Implementation and tests: `internal/stage/merge.go`,
|
||||
`internal/stage/merge_test.go`
|
||||
34
docs/internal/stage-normalize.md
Normal file
34
docs/internal/stage-normalize.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Stage: normalize
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.json` (or configured normalize output path)
|
||||
- optional `artifacts/seriatim.normalize.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
- validates normalized transcript and optional report.
|
||||
- materializes canonical outputs and records logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- final transcript must validate as processed transcript JSON (`segments` array).
|
||||
- normalize defaults are applied when `pipeline.normalize` is unset.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns normalize fields and defaults.
|
||||
- Implementation and tests: `internal/stage/normalize.go`,
|
||||
`internal/stage/normalize_test.go`
|
||||
36
docs/internal/stage-polish.md
Normal file
36
docs/internal/stage-polish.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Stage: polish
|
||||
|
||||
## Purpose
|
||||
|
||||
Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- `inputs/glossary.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
- optional `artifacts/audita.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves base transcript from merge outputs/canonical fallback.
|
||||
- invokes Audita with configured model/module/runtime options.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
- validates optional report JSON.
|
||||
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||
|
||||
## Invariants
|
||||
|
||||
- polished transcript schema validation is mandatory.
|
||||
- report output is config-gated.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Audita](../integrations/audita.md) owns subprocess, validation, and failure
|
||||
semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Audita values.
|
||||
- Implementation and tests: `internal/stage/polish.go`,
|
||||
`internal/stage/polish_test.go`
|
||||
60
docs/internal/stage-prepare.md
Normal file
60
docs/internal/stage-prepare.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Stage: prepare
|
||||
|
||||
## Purpose
|
||||
|
||||
Materialize canonical current-session inputs before processing stages.
|
||||
|
||||
## Inputs
|
||||
|
||||
- resolved campaign, session, and pipeline configuration
|
||||
- stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
|
||||
- one resolved local or S3 audio source
|
||||
- enabled configured artifact input requirements for previous-session sources
|
||||
|
||||
## Outputs
|
||||
|
||||
- `inputs/campaign.yml`
|
||||
- `inputs/session.yml`
|
||||
- `inputs/pipeline.resolved.yml`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
- `inputs/glossary.yml`
|
||||
- `inputs/players.yml`
|
||||
- `inputs/party.yml`
|
||||
- `audio/*.flac`
|
||||
- optional `previous/manifest.json`
|
||||
- optional `previous/artifacts/**`
|
||||
- deterministic `manifest.inputs` entries (checksums + provenance)
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- validates required config/store state.
|
||||
- enforces local audio vs S3 audio mutual exclusivity.
|
||||
- materializes S3 audio through spool/cache-aware logic.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- when previous requirements exist:
|
||||
- clears managed `previous/` state;
|
||||
- builds previous-cache remote plan;
|
||||
- downloads previous manifest/artifacts;
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs are skipped.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `prepare` hydrates canonical `previous/` cache state.
|
||||
- managed previous artifacts are stored under `previous/artifacts/**` without
|
||||
duplicate `artifacts/artifacts/` nesting.
|
||||
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md) owns audio selection, stable input fields, and
|
||||
previous-session settings.
|
||||
- [Operations](../operations.md) owns physical input, audio, spool, cache, and
|
||||
previous-state layout.
|
||||
- [Storage Internals](storage.md) and [Artifact Internals](artifacts.md) explain
|
||||
the internal collaborators.
|
||||
- Implementation and tests: `internal/stage/prepare.go`,
|
||||
`internal/stage/prepare_test.go`, `internal/audio/s3_audio_test.go`,
|
||||
`internal/previouscache/*_test.go`
|
||||
74
docs/internal/stage-publish.md
Normal file
74
docs/internal/stage-publish.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Stage: publish
|
||||
|
||||
## Purpose
|
||||
|
||||
Upload run/session outputs to object storage and atomically advance remote current state.
|
||||
|
||||
## Inputs
|
||||
|
||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||
- invocation-scoped run files
|
||||
- resolved publish output rules
|
||||
- effective publish locks (static + remote merged lock set)
|
||||
- durable previous-session cache files when present
|
||||
|
||||
## Outputs
|
||||
|
||||
- uploaded invocation record and selected publish outputs;
|
||||
- uploaded durable previous-session cache files when present;
|
||||
- updated remote current manifest; and
|
||||
- remote current-run commit marker, written last.
|
||||
|
||||
Exact remote placement and the operator workflow belong in
|
||||
[Operations](../operations.md#publish-workflow).
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- stage can self-skip when publish disabled or run upload disabled.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- collects a deterministic run file list plus run `manifest.json`, excluding
|
||||
`audio/**` and the run-local `extract/notarius-output/**` staging bundle.
|
||||
- keeps run-local Notarius receipt and stderr diagnostics eligible for the run
|
||||
archive.
|
||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
- writes remote current manifest before current run pointer.
|
||||
|
||||
## Metadata Signals
|
||||
|
||||
Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
- previous uploads
|
||||
- skipped optional outputs
|
||||
- skipped unselected outputs
|
||||
- locked outputs
|
||||
- current-state key paths
|
||||
- `current_pointer_written`
|
||||
|
||||
## Invariants
|
||||
|
||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||
- run upload excludes `audio/**` and `extract/notarius-output/**`.
|
||||
- `extract/notarius.receipt.json` and `extract/notarius.stderr.log` remain
|
||||
eligible run-record diagnostics.
|
||||
- publish locks are not overridden by `--force`.
|
||||
|
||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output and
|
||||
static-lock fields.
|
||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||
physical remote state.
|
||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||
helpers.
|
||||
- Implementation and tests: `internal/stage/publish.go`,
|
||||
`internal/stage/publish_test.go`, `internal/app/operator_helpers_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
42
docs/internal/stage-render.md
Normal file
42
docs/internal/stage-render.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Stage: render
|
||||
|
||||
## Purpose
|
||||
|
||||
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `narratio.transcript.final` (`transcripts/final.json`)
|
||||
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
|
||||
|
||||
## Outputs
|
||||
|
||||
- `narratio.transcript.final_markdown` -> `transcripts/final.md`
|
||||
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||
- resolves inputs manifest-first, then canonical fallback.
|
||||
- writes run-local outputs first, then materializes canonical session outputs.
|
||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||
- skips with stage metadata when `pipeline.render.enabled=false`.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- missing normalized input fails with normalize rerun guidance.
|
||||
- missing trimmed input fails with trim rerun guidance.
|
||||
- adapter/subprocess failure fails stage.
|
||||
- empty render output files fail validation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `format: markdown` is supported.
|
||||
- render stage owns production of built-in Markdown transcript sources.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns render subprocess behavior.
|
||||
- [Configuration](../config.md#pipeline) owns render fields and defaults.
|
||||
- Implementation and tests: `internal/stage/render.go`,
|
||||
`internal/stage/render_test.go`
|
||||
36
docs/internal/stage-transcribe.md
Normal file
36
docs/internal/stage-transcribe.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Stage: transcribe
|
||||
|
||||
## Purpose
|
||||
|
||||
Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `audio/*.flac` from `prepare`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/raw/<speaker>.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||
- derives speaker ID from `.flac` basename.
|
||||
- dispatches WhisperX requests through a bounded worker pool.
|
||||
- validates each output as JSON.
|
||||
- writes run-local outputs then materializes canonical transcript outputs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- speaker basenames must be unique.
|
||||
- output path returned by adapter must match requested output path.
|
||||
- each successful output is validated before stage success.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [WhisperX](../integrations/whisperx.md) owns HTTP, retry, timeout, and
|
||||
cancellation semantics.
|
||||
- [Configuration](../config.md#pipeline) owns concurrency and other
|
||||
operator-selected values.
|
||||
- Implementation and tests: `internal/stage/transcribe.go`,
|
||||
`internal/stage/transcribe_test.go`
|
||||
43
docs/internal/stage-trim.md
Normal file
43
docs/internal/stage-trim.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Stage: trim
|
||||
|
||||
## Purpose
|
||||
|
||||
Produce a final-trimmed transcript. By default, the stage generates bounds and
|
||||
applies a bounds-driven trim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/final.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.trimmed.json` (or configured trim output path)
|
||||
- when trim enabled: `artifacts/session_bounds.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
When `trim.enabled=true`:
|
||||
- runs Scriptorium bounds artifact generation;
|
||||
- optionally runs render-debug output generation;
|
||||
- validates bounds payload against transcript;
|
||||
- derives keep selector;
|
||||
- either copies unchanged transcript or runs Seriatim trim;
|
||||
- validates trimmed transcript and materializes bounds output.
|
||||
|
||||
When `trim.enabled=false`:
|
||||
- copies normalized transcript to trimmed output.
|
||||
|
||||
## Invariants
|
||||
|
||||
- normalized transcript is required input.
|
||||
- bounds output exists only in enabled trim path.
|
||||
- render-debug output is diagnostic and not a declared stage output.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns bounds generation and
|
||||
debug-render subprocess behavior.
|
||||
- [Seriatim](../integrations/seriatim.md) owns transcript trimming behavior.
|
||||
- [Configuration](../config.md#pipeline) owns trim fields and defaults.
|
||||
- Implementation and tests: `internal/stage/trim.go`,
|
||||
`internal/stage/trim_test.go`
|
||||
48
docs/internal/storage.md
Normal file
48
docs/internal/storage.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Internal: Storage
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the object-store interface and S3 implementation used by Narratio.
|
||||
Remote key layout and lifecycle belong in [Operations](../operations.md), while
|
||||
operator-selected storage fields and credential mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
## Primary Contract
|
||||
|
||||
`storage.ObjectStore` interface:
|
||||
|
||||
- `List(ctx, prefix)`
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
- callers pass full bucket-relative keys;
|
||||
- storage implementations do not infer campaign/session/run prefixes.
|
||||
|
||||
## Composition
|
||||
|
||||
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
|
||||
resolved configuration. The application loads configured filesystem secrets
|
||||
before calling it. The storage adapter consumes already-resolved values; it does
|
||||
not own discovery, defaults, or configuration validation.
|
||||
|
||||
## S3 Backend Behavior
|
||||
|
||||
- normalizes object keys.
|
||||
- `List` paginates and returns normalized `ObjectInfo`.
|
||||
- `Download` writes local files with parent directory creation.
|
||||
- `Upload` streams local file and returns remote metadata.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- storage layer is stateless regarding manifest/stage progression.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Contract and S3 adapter: `internal/adapters/storage`
|
||||
- Composition: `internal/app/object_store.go`
|
||||
- Tests: `internal/adapters/storage/*_test.go`,
|
||||
`internal/app/object_store_test.go`
|
||||
74
docs/internal/workspace.md
Normal file
74
docs/internal/workspace.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Internal: Workspace
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the helpers that construct local session and run paths, coordinate
|
||||
single-writer access, and confine cleanup. The authoritative physical layout and
|
||||
retention workflow belong in [Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Path Ownership
|
||||
|
||||
`internal/artifacts` owns canonical session, run, spool, cache, and
|
||||
previous-cache path construction. `SessionPathsFor` provides the session-scoped
|
||||
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||
consume those helpers instead of rebuilding relative paths.
|
||||
|
||||
`internal/pathsafe` and application cleanup helpers enforce confinement for
|
||||
relative destinations and deletion targets.
|
||||
|
||||
## Run-Local Stage Layout
|
||||
|
||||
`internal/stage/run_local.go` maps stage outputs and diagnostics into an
|
||||
invocation-scoped layout. Successful outputs are validated and atomically
|
||||
materialized into canonical session paths before stage success. Managed
|
||||
previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||
copies only regular files and directories to a same-filesystem temporary
|
||||
sibling. Source traversal uses confined directory handles and identity checks
|
||||
so replacing an inspected root, directory, or file is rejected rather than
|
||||
followed. The completed tree is atomically renamed without replacing an
|
||||
existing destination. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
## Locking
|
||||
|
||||
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
||||
(`ErrLockConflict` on contention).
|
||||
|
||||
## Cleanup Semantics
|
||||
|
||||
Automatic post-publish cleanup:
|
||||
|
||||
- only runs when publish actually executed and succeeded;
|
||||
- requires `uploaded=true` and `current_pointer_written=true` metadata;
|
||||
- consumes the resolved cleanup policy described in
|
||||
[Configuration](../config.md);
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
|
||||
|
||||
Manual cleanup uses the same scoped-target checks. Invocation syntax and exact
|
||||
deletion scope belong in [CLI](../cli.md#clean) and
|
||||
[Operations](../operations.md#cleanup).
|
||||
|
||||
## Invariants
|
||||
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
||||
`internal/app/post_publish_cleanup.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/directory_test.go`,
|
||||
`internal/app/cleanup_targets_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
322
docs/operations.md
Normal file
322
docs/operations.md
Normal file
@@ -0,0 +1,322 @@
|
||||
# Operations Guide
|
||||
|
||||
Operator workflow for running, recovering, and publishing Narratio sessions.
|
||||
|
||||
For command syntax, see [docs/cli.md](./cli.md). For field-level config, see [docs/config.md](./config.md).
|
||||
|
||||
## Campaign and Session Selection
|
||||
|
||||
Campaign selection priority:
|
||||
|
||||
- `--campaign-file`
|
||||
- `--campaign`
|
||||
- `pipeline.campaigns.default_campaign_id`
|
||||
|
||||
Session source priority:
|
||||
|
||||
- `--session`
|
||||
- local default search paths
|
||||
- remote session object (S3) when local session file is not found and storage is configured
|
||||
|
||||
## Session Initialization
|
||||
|
||||
Use `session init` to generate a concrete session file for local or remote use.
|
||||
|
||||
Local file:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Remote session object:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --remote --force
|
||||
```
|
||||
|
||||
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
|
||||
|
||||
Campaigns must provide stable input files for speakers, autocorrect, glossary, players, and party. Session files may override those paths for one session. The `prepare` stage materializes them under `inputs/`; configured Scriptorium artifacts can reference prepared `players`, `party`, and `glossary` files with `narratio.input.players`, `narratio.input.party`, and `narratio.input.glossary`.
|
||||
|
||||
## Standard Session Workflow
|
||||
|
||||
1. Select pipeline/campaign/session config.
|
||||
2. Validate session readiness:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
3. (Optional) inspect stage decisions:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
4. Run the pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
5. Check state:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
## Stage Execution and Continuation Behavior
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `extract`
|
||||
8. `render`
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- forcing an upstream stage marks succeeded downstream stages as `stale` before
|
||||
the replacement runs; and
|
||||
- an executed failure, changed self-skip, or success that replaces a different
|
||||
effective upstream outcome also marks succeeded downstream stages stale. A
|
||||
repeated self-skip with the same reason and no outputs is stable and does not
|
||||
perpetually rerun downstream work.
|
||||
|
||||
Single-stage execution:
|
||||
|
||||
```bash
|
||||
narratio run-stage normalize 2026-04-04 --force
|
||||
```
|
||||
|
||||
## Artifact Selection
|
||||
|
||||
`--artifacts` can be used on `run`, `run-stage`, `analyze`, and `publish`.
|
||||
|
||||
Selection behavior:
|
||||
|
||||
- validates names against `pipeline.scriptorium.artifacts`;
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- never partially selects Notarius lanes.
|
||||
|
||||
## Extraction Workflow
|
||||
|
||||
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||
|
||||
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||
and preserves the complete validated Notarius bundle at:
|
||||
|
||||
- `artifacts/notarius/{narratio_run_id}/`
|
||||
|
||||
The directory is immutable once promoted. Configured lanes become
|
||||
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||
the bundle and `index.json` are retained for audit and resume validation but
|
||||
are not selectable or published implicitly.
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||
immutable bundle remain available for inspection, but downstream resolution
|
||||
requires a new current successful extraction record.
|
||||
|
||||
Atomic Notarius bundle promotion is supported on Linux, macOS, and Windows.
|
||||
On other operating systems, extraction fails before copying the bundle into a
|
||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||
directory primitive there. This is an extraction limitation, not a broader
|
||||
platform-support guarantee for every Narratio workflow.
|
||||
|
||||
Run-local diagnostics are:
|
||||
|
||||
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run-record upload excludes the complete
|
||||
`extract/notarius-output/**` subtree. The receipt and stderr files remain
|
||||
eligible run-record diagnostics. The durable bundle is never scanned for
|
||||
implicit publication; only lanes named by explicit `pipeline.publish.outputs`
|
||||
rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio automatically reruns extraction when its recorded invocation contract
|
||||
or durable output validation changes. It cannot fingerprint configuration
|
||||
files, profiles, prompts, modules, or references loaded transitively by
|
||||
Notarius. Force extraction after changing any of those inputs, even when the
|
||||
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||
marks successful downstream stages stale. Ordinary extraction failures or
|
||||
outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
Run publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
Equivalent:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
audio and the run-local Notarius staging bundle;
|
||||
- uploads configured published outputs, including only explicitly configured
|
||||
extraction lanes;
|
||||
- uploads `previous/**` cache files when present;
|
||||
- writes `current/manifest.json`;
|
||||
- writes `current/run_id.txt` last.
|
||||
|
||||
`current/run_id.txt` is the remote current-state commit marker.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
Lock sources:
|
||||
|
||||
- static locks in `pipeline.publish.locks`
|
||||
- mutable remote locks in `{session_prefix}/locks.yml`
|
||||
|
||||
Effective lock rules:
|
||||
|
||||
- static and remote locks are merged;
|
||||
- static locks win on source collisions;
|
||||
- locked outputs are intentional skips;
|
||||
- lock add/remove commands mutate only remote lock state.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
narratio session locks 2026-04-04
|
||||
narratio session locks add 2026-04-04 narratio.artifact.session_recap --reason "manual edits" --force
|
||||
narratio session locks remove 2026-04-04 narratio.artifact.session_recap
|
||||
```
|
||||
|
||||
## Restore Workflow
|
||||
|
||||
Use restore when local durable session state is missing or stale and remote committed current state is authoritative.
|
||||
|
||||
Dry run:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Apply:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- `manifest.json`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**` when needed by configured previous-session artifact inputs
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite local conflicts
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
## Local State Layout
|
||||
|
||||
Session root:
|
||||
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
|
||||
Durable session paths:
|
||||
|
||||
- `manifest.json`
|
||||
- `inputs/**`
|
||||
- `audio/**`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**`
|
||||
- `reports/**`
|
||||
- `logs/**`
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
|
||||
Run-local layout:
|
||||
|
||||
- `runs/{run_id}/{stage}/outputs`
|
||||
- `runs/{run_id}/{stage}/logs`
|
||||
- `runs/{run_id}/{stage}/reports`
|
||||
- `runs/{run_id}/{stage}/config`
|
||||
- `runs/{run_id}/{stage}/scratch`
|
||||
|
||||
Spool layout (runtime/transient):
|
||||
|
||||
- `{spool.root}/{campaign}/{session_id}/{run_id}/...`
|
||||
- restore audio spool under `{spool.root}/{campaign}/{session_id}/restore/audio`
|
||||
|
||||
Cache layout (durable S3 audio cache):
|
||||
|
||||
- `{cache.root}/s3/{bucket}/...`
|
||||
|
||||
## Cleanup
|
||||
|
||||
Session-scoped cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04
|
||||
```
|
||||
|
||||
Global cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean --all
|
||||
```
|
||||
|
||||
Dry-run and cache variants:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04 --dry-run --clear-cache
|
||||
narratio clean --all --dry-run --clear-cache
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
- Local and S3 audio modes are mutually exclusive.
|
||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||
- Restore requires configured object storage and committed remote current state.
|
||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
||||
241
docs/policy/architecture.md
Normal file
241
docs/policy/architecture.md
Normal file
@@ -0,0 +1,241 @@
|
||||
# Architecture
|
||||
|
||||
This document defines Narratio's intended high-level architecture and the
|
||||
invariants that changes must preserve. Implemented component details belong in
|
||||
the [Internal Overview](../internal/overview.md) and its linked documents.
|
||||
Significant architectural decision history belongs under `docs/adr/` when such
|
||||
records exist.
|
||||
|
||||
## System Shape
|
||||
|
||||
Narratio is a small Go application that turns D&D session audio into polished
|
||||
transcripts and generated session artifacts. It is an explicit, stage-driven
|
||||
orchestrator, not a general workflow engine.
|
||||
|
||||
Narratio coordinates specialized external systems rather than reimplementing
|
||||
their domains:
|
||||
|
||||
- WhisperX performs transcription;
|
||||
- Seriatim performs deterministic transcript processing and rendering;
|
||||
- Audita performs transcript correction and polishing;
|
||||
- Notarius extracts validated structured artifact bundles; and
|
||||
- Scriptorium executes prompts and produces configured artifacts.
|
||||
|
||||
Narratio owns orchestration, configuration resolution, session and run state,
|
||||
artifact and path modeling, manifest persistence, stage sequencing, resume,
|
||||
restore, cleanup gates, and publish semantics. External contracts are defined
|
||||
in the [integration documentation](../integrations/).
|
||||
|
||||
The pipeline has one canonical ordered stage set. Configuration may enable,
|
||||
disable, or parameterize supported behavior, but it must not turn that sequence
|
||||
into an arbitrary DAG or hide orchestration in generic workflow abstractions.
|
||||
The implemented stage inventory belongs in the
|
||||
[Internal Overview](../internal/overview.md).
|
||||
|
||||
Narratio is contract-first without being abstraction-heavy. Interfaces and
|
||||
extension points should protect demonstrated boundaries. New abstraction is not
|
||||
itself an architectural goal.
|
||||
|
||||
## Ownership And Dependency Direction
|
||||
|
||||
The application boundary owns command dispatch, configuration selection,
|
||||
production composition, session locking, and top-level lifecycle. It may depend
|
||||
on concrete implementations to assemble a run.
|
||||
|
||||
Stage orchestration expresses intent in Narratio-level data and interfaces.
|
||||
Stages may depend on configuration, manifest, artifact, path, and adapter
|
||||
contracts, but they must not depend on transport-specific request types,
|
||||
subprocess argument construction, cloud SDK types, or downstream tool internals.
|
||||
|
||||
Adapters translate between Narratio contracts and external systems. They own
|
||||
HTTP, subprocess, notification, and object-storage mechanics, including command
|
||||
construction, transport behavior, provider response handling, and external
|
||||
error adaptation. External dependency types must remain inside the adapter that
|
||||
owns them unless that dependency is the adapter's explicit public contract.
|
||||
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||
construction, notification transport, and object-storage SDK details remain
|
||||
behind these boundaries.
|
||||
|
||||
State and path services must not infer stage policy. Storage implementations
|
||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||
or root-prefix semantics. Manifest persistence records transitions but does not
|
||||
choose orchestration policy. Artifact resolution identifies and validates
|
||||
artifacts but does not execute producers.
|
||||
|
||||
Dependencies should remain narrow and point toward Narratio-owned contracts.
|
||||
Prefer the Go standard library. Add an external dependency only when it provides
|
||||
a clear correctness, security, interoperability, or complexity benefit, and
|
||||
confine it to the boundary that needs it.
|
||||
|
||||
## Stage Boundaries
|
||||
|
||||
Each stage has one explicit responsibility and declares:
|
||||
|
||||
- required input state;
|
||||
- produced output state;
|
||||
- configuration it consumes;
|
||||
- external adapters it uses;
|
||||
- manifest references and metadata it reads or writes;
|
||||
- skip, force, invalidation, and resume behavior; and
|
||||
- failure behavior.
|
||||
|
||||
Stages write and validate run-local results before materializing canonical
|
||||
outputs where that distinction applies. A stage is complete only after its
|
||||
required outputs have been written, validated, and recorded in durable manifest
|
||||
state. Later stages depend on recorded success and artifact resolution, not
|
||||
merely on incidental files existing on disk.
|
||||
|
||||
A failed or interrupted stage must not be presented as successful. Failure
|
||||
should preserve enough local state and diagnostics for inspection, recovery,
|
||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||
according to the canonical stage order.
|
||||
|
||||
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||
outcome is persisted, clears older outputs owned by the stage, and is
|
||||
reconsidered on a later invocation. A stage may also validate whether an
|
||||
otherwise successful recorded result is still resumable; an obsolete result
|
||||
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||
stops execution.
|
||||
|
||||
Shared behavior should live behind a narrow service or helper with one clear
|
||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||
artifact, or path policy ad hoc.
|
||||
|
||||
## Manifest, Resume, And Restore
|
||||
|
||||
The session manifest is the durable ledger for progress across invocations. It
|
||||
records session and run identity, stage state, input and output references,
|
||||
diagnostic references, checksums or provenance where useful, and non-secret
|
||||
adapter and publish metadata.
|
||||
|
||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||
inspected and validated, but file presence alone does not replace recorded
|
||||
stage state. Invocation-scoped run records provide an audit of one execution;
|
||||
they do not replace the session manifest as progress authority.
|
||||
|
||||
Restore treats committed remote current state as its authority. It must plan
|
||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||
install the validated session manifest after other restored durable files. The
|
||||
physical workflow and recovery procedures belong in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration is strict, explicit, centralized, and operator-oriented.
|
||||
|
||||
- YAML decoding rejects unknown fields.
|
||||
- Defaults are centralized and testable.
|
||||
- Empty configured values do not silently replace meaningful defaults.
|
||||
- Validation rejects invalid composition before stage execution where
|
||||
practical.
|
||||
- Session templating remains narrow and deterministic rather than becoming a
|
||||
general configuration language.
|
||||
- Secret values are supplied indirectly and are not persisted in ordinary
|
||||
configuration.
|
||||
|
||||
Narratio must not become a second configuration system for downstream tools.
|
||||
External systems own their runtime defaults wherever practical; Narratio passes
|
||||
the paths required by its stage contracts and explicit operator overrides. The
|
||||
field-level contract and credential-supply mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
## Artifacts, Paths, And Storage
|
||||
|
||||
Artifact identities and local and remote paths are application contracts.
|
||||
Canonical helpers own workspace, spool, cache, session, run, input, transcript,
|
||||
artifact, log, report, configuration, and publish-current paths. Callers must
|
||||
not reconstruct canonical paths through scattered string concatenation.
|
||||
|
||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||
canonical outputs before reporting success, and consumers resolve declared
|
||||
artifact identities rather than infer files from unrelated directory contents.
|
||||
External artifact bundles become current only through validated immutable
|
||||
promotion and manifest records; directory presence alone never establishes
|
||||
availability.
|
||||
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
relative destinations must not expand the scope of an operation. Cleanup is
|
||||
permitted only through explicit operator action or configured post-publish
|
||||
gates, and it must preserve durable cache unless cache removal is explicitly
|
||||
requested.
|
||||
|
||||
Physical layout, retention, and operational lifecycle belong in
|
||||
[Operations](../operations.md). Logical external formats and durable integration
|
||||
contracts belong under [Integrations](../integrations/).
|
||||
|
||||
## Publish Commit Boundary
|
||||
|
||||
Publish has one explicit remote commit boundary. A remote run becomes current
|
||||
only after Narratio has successfully uploaded the run record, required published
|
||||
outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
||||
|
||||
`current/run_id.txt` is the commit marker and must be written last. Failed,
|
||||
incomplete, skipped, or uncommitted publish attempts must not be presented as
|
||||
current remote state. Publish locks remain authoritative and are not bypassed by
|
||||
a forced run.
|
||||
|
||||
Automatic local cleanup is permitted only after a successful publish commit,
|
||||
only when explicitly configured, and only through the path-safety guardrails.
|
||||
|
||||
## Security, Privacy, And Diagnostics
|
||||
|
||||
Narratio handles private campaign material. Transcripts, prompts, generated
|
||||
artifacts, reports, logs, manifests, and diagnostic files are potentially
|
||||
sensitive.
|
||||
|
||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||
to manifests, logs, generated configuration, reports, publish metadata,
|
||||
documentation, or examples. Secrets enter through configured environment
|
||||
variable names or secret-file references. Diagnostics should avoid transcript
|
||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
||||
it.
|
||||
|
||||
Logs, reports, generated invocation files, generated configuration, and render
|
||||
debug files are diagnostics, not canonical pipeline products. They should be
|
||||
durable and discoverable where configured, and manifest references must preserve
|
||||
the distinction between diagnostics and artifacts.
|
||||
|
||||
Documentation security rules belong in the
|
||||
[Documentation Policy](documentation.md). Credential supply belongs in
|
||||
[Configuration](../config.md), while permissions, sensitive runtime-artifact
|
||||
handling, and recovery belong in [Operations](../operations.md).
|
||||
|
||||
## Determinism And Testability
|
||||
|
||||
Narratio prefers deterministic behavior where practical, including stable local
|
||||
and remote layouts, sorted operation order, predictable generated
|
||||
configuration, repeatable command construction, deterministic artifact
|
||||
resolution, and reproducible planning.
|
||||
|
||||
Run IDs and timestamps may be intentionally variable, but surrounding behavior
|
||||
must remain controllable in tests. Core behavior should be testable without live
|
||||
external services; expensive, nondeterministic, destructive, or external
|
||||
boundaries should be replaceable with focused test doubles. General testing
|
||||
philosophy and sufficiency rules belong in the [Testing Policy](testing.md).
|
||||
|
||||
## Documentation And Decision Records
|
||||
|
||||
Documentation follows the [Documentation Policy](documentation.md). Current
|
||||
behavior belongs in its canonical user, operator, integration, architecture, or
|
||||
internal owner. Proposed behavior and implementation status belong under
|
||||
`docs/roadmap/`.
|
||||
|
||||
Significant architectural decisions may be recorded under `docs/adr/` using the
|
||||
format and lifecycle defined by the documentation policy. ADR acceptance does
|
||||
not establish that a decision has been implemented.
|
||||
|
||||
## Architectural Non-Goals
|
||||
|
||||
Narratio does not aim to provide:
|
||||
|
||||
- a generic DAG or workflow engine;
|
||||
- a replacement configuration layer for WhisperX, Seriatim, Audita,
|
||||
Scriptorium, or other downstream tools;
|
||||
- a storage abstraction broader than the needs of this pipeline;
|
||||
- stage logic coupled directly to cloud SDKs, transports, subprocess details,
|
||||
or downstream implementation internals;
|
||||
- raw-secret persistence;
|
||||
- implicit cross-stage behavior that bypasses manifest and artifact contracts;
|
||||
or
|
||||
- a prompt-authoring system.
|
||||
148
docs/policy/documentation.md
Normal file
148
docs/policy/documentation.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep Narratio documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
stage or integration keys, schemas, file names, paths, status codes, retry
|
||||
behavior, and runtime guarantees. If readers could reasonably treat a statement
|
||||
as a contract, maintain it only in the owning document.
|
||||
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, output conventions, and exit behavior. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, stage implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schemas, fields, defaults, environment overrides, validation rules, and user-selectable stage or integration settings. | Complete example files, CLI syntax, runtime state lifecycle, implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and remote-state layout, output and diagnostic handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical artifact schemas, implementation mechanics. |
|
||||
| Troubleshooting | `docs/troubleshooting.md` | Symptom-driven diagnosis, likely causes, safe inspection steps and remedies, and links to relevant contracts. | CLI syntax, configuration definitions, operational procedures, integration contracts, implementation mechanics. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical artifact paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, Operations, And Troubleshooting
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. Troubleshooting
|
||||
starts from observable symptoms and links readers to the owning command,
|
||||
configuration, operational, or integration contract. When a workflow crosses
|
||||
these topics, choose the document that owns the task and link to the other
|
||||
contracts.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how Narratio implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Troubleshooting owns safe diagnostic and remediation guidance.
|
||||
Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
296
docs/policy/testing.md
Normal file
296
docs/policy/testing.md
Normal file
@@ -0,0 +1,296 @@
|
||||
# Testing Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
4816
docs/roadmap/audit-findings.md
Normal file
4816
docs/roadmap/audit-findings.md
Normal file
File diff suppressed because it is too large
Load Diff
332
docs/roadmap/audit-plan.md
Normal file
332
docs/roadmap/audit-plan.md
Normal file
@@ -0,0 +1,332 @@
|
||||
# Codebase Audit Plan
|
||||
|
||||
Status: proposed
|
||||
|
||||
## Purpose
|
||||
|
||||
This audit will evaluate Narratio for correctness, efficiency, maintainability,
|
||||
and test-suite value. It will identify defects and credible risks, duplicated or
|
||||
near-duplicated behavior, code that can be made smaller or more idiomatic, and
|
||||
complex code whose remaining invariants need focused explanation.
|
||||
|
||||
The audit is investigative. It should produce evidence-backed findings and a
|
||||
prioritized remediation backlog, not make opportunistic production changes as
|
||||
it proceeds. The [Audit Sequence](audit-sequence.md) assigns this scope to
|
||||
concrete execution stages.
|
||||
|
||||
## Authoritative Baseline
|
||||
|
||||
Review implemented behavior against its canonical owner rather than treating
|
||||
the current implementation or tests as the specification:
|
||||
|
||||
- [Architecture](../policy/architecture.md) for system boundaries, dependency
|
||||
direction, state and path ownership, safety properties, and pipeline
|
||||
invariants;
|
||||
- [Internal Overview](../internal/overview.md) and its focused internal
|
||||
documents for implemented ownership and mechanics;
|
||||
- [Testing Policy](../policy/testing.md) for risk-based sufficiency, durable
|
||||
boundaries, test-double guidance, and test lifecycle decisions;
|
||||
- the [CLI](../cli.md), [Configuration](../config.md),
|
||||
[Operations](../operations.md), and [integration contracts](../integrations/)
|
||||
for externally observable behavior; and
|
||||
- the [Documentation Policy](../policy/documentation.md) for canonical ownership
|
||||
and the distinction between current and proposed behavior.
|
||||
|
||||
Where code, tests, and documentation disagree, record the disagreement. Do not
|
||||
assume which one is wrong until the canonical contract and caller expectations
|
||||
have been traced.
|
||||
|
||||
## Audit Principles
|
||||
|
||||
1. Review correctness before cleanup. A shorter implementation is not an
|
||||
improvement if it weakens a state transition, safety check, or external
|
||||
contract.
|
||||
2. Trace behavior across boundaries. Narratio's most important properties often
|
||||
emerge from the interaction of application orchestration, stages, manifests,
|
||||
artifact resolution, filesystem operations, and adapters.
|
||||
3. Distinguish repeated syntax from repeated policy. Extract a helper only when
|
||||
the behavior has one stable owner and the shared abstraction makes that
|
||||
ownership clearer. Similar stage code may be intentionally explicit.
|
||||
4. Prefer narrow, idiomatic Go over generic frameworks. In particular, proposed
|
||||
refactors must preserve the explicit canonical stage sequence and must not
|
||||
turn Narratio into a workflow engine or a second configuration system for
|
||||
downstream tools.
|
||||
5. Optimize credible work. Flag repeated I/O, hashing, serialization, remote
|
||||
calls, subprocess work, allocation, or poor asymptotic behavior when the
|
||||
relevant path can matter. Require a benchmark or workload argument for
|
||||
performance changes whose benefit is not evident.
|
||||
6. Treat comments as explanations of intent. Recommend comments for invariants,
|
||||
ordering constraints, non-obvious failure policy, or security reasoning—not
|
||||
as narration of ordinary Go or a substitute for simplifying code.
|
||||
7. Judge tests as a suite. A test can be locally reasonable and still add no
|
||||
marginal protection, while a compact test can be inadequate for a
|
||||
consequential cross-component failure.
|
||||
|
||||
## Evidence And Finding Standard
|
||||
|
||||
Begin from a cleanly identified revision and record toolchain and platform
|
||||
assumptions. Use the code knowledge graph to find ownership, callers, callees,
|
||||
similarity candidates, high-complexity functions, and weakly protected
|
||||
boundaries. Confirm every candidate by reading the implementation, its focused
|
||||
tests, and the applicable contract. Text search and static analysis supplement
|
||||
the graph for literals, configuration, generated files, and patterns that are
|
||||
not modeled reliably.
|
||||
|
||||
Each finding should record:
|
||||
|
||||
- category: correctness defect, correctness risk, duplication, simplification,
|
||||
efficiency, architectural boundary, comment/clarity, or test-suite issue;
|
||||
- source locations and the affected contract or invariant;
|
||||
- concrete evidence and a realistic failure or maintenance scenario;
|
||||
- impact, likelihood, confidence, and estimated remediation scope separately;
|
||||
- the smallest plausible improvement and its intended owner;
|
||||
- tests that already protect the behavior, tests that should change or be
|
||||
added, and tests that may become redundant; and
|
||||
- dependencies on, or conflicts with, other findings.
|
||||
|
||||
Do not report a metric alone as a finding. Complexity, similarity, coverage,
|
||||
fan-in, file size, and test count are prioritization signals that require manual
|
||||
confirmation. Consolidate findings that share one root cause.
|
||||
|
||||
## Cross-Cutting Review Lenses
|
||||
|
||||
### Correctness And Pipeline Semantics
|
||||
|
||||
Construct an explicit lifecycle matrix for every stage outcome: first run,
|
||||
already-succeeded skip, self-skip, failure, interruption, forced replacement,
|
||||
non-resumable result, and successful rerun. Trace how each outcome changes the
|
||||
session manifest, invocation manifest, downstream stage state, artifacts,
|
||||
diagnostics, and cleanup eligibility.
|
||||
|
||||
Across the pipeline, verify:
|
||||
|
||||
- the registry exposes one deterministic canonical order;
|
||||
- each stage's declared inputs, outputs, configuration, adapters, and manifest
|
||||
effects agree with its implementation and focused documentation;
|
||||
- inputs are resolved through manifest and artifact contracts rather than
|
||||
incidental directory contents;
|
||||
- run-local outputs are fully validated before canonical materialization;
|
||||
- failure, cancellation, or process interruption cannot advertise partial work
|
||||
as successful;
|
||||
- force and changed outcomes invalidate exactly the intended succeeded
|
||||
downstream work;
|
||||
- repeated execution is idempotent where promised, and ordering is stable
|
||||
wherever maps, directory reads, remote listings, or dependency graphs are
|
||||
involved;
|
||||
- session, campaign, run, source, checksum, contract, and external provenance
|
||||
identities cannot be confused across runs; and
|
||||
- errors preserve useful causes and do not expose secrets or private content.
|
||||
|
||||
Use fault-oriented reasoning at durability boundaries: fail immediately before
|
||||
and after manifest saves, canonical renames, external process completion,
|
||||
uploads, current-manifest publication, the current-run commit marker, restore
|
||||
manifest installation, and cleanup. Determine which state is authoritative and
|
||||
whether the next invocation recovers safely.
|
||||
|
||||
### Duplication And Helper Ownership
|
||||
|
||||
Search for exact and semantic duplication in production and tests, including:
|
||||
|
||||
- repeated stage setup, input resolution, output validation, run-local
|
||||
materialization, metadata construction, and error adaptation;
|
||||
- repeated manifest create/load/save and session/run transition handling;
|
||||
- repeated adapter construction, timeout parsing, command execution, generated
|
||||
configuration, log handling, and output checks;
|
||||
- repeated source-ID, destination, remote-key, and path validation policy;
|
||||
- repeated sorting, deduplication, checksum, copy, and atomic-write mechanics;
|
||||
and
|
||||
- repeated test fixtures and assertions that encode the same policy at several
|
||||
layers.
|
||||
|
||||
For each candidate, decide whether it is coincidental similarity, a repeated
|
||||
mechanism, or duplicated policy. Recommend extraction only when the helper can
|
||||
have a clear package owner, a narrow contract, and callers that become easier
|
||||
to understand. Prefer an unexported local helper when sharing is package-local.
|
||||
Do not create a broad utility package, force unlike stage results into one data
|
||||
model, or move policy into storage/file-operation helpers.
|
||||
|
||||
Initial similarity and complexity signals should seed, but not predetermine,
|
||||
inspection of the single-stage command wrappers, session/run manifest
|
||||
persistence pairs, adapter constructors, Scriptorium operations, stage fakes,
|
||||
and common stage materialization paths.
|
||||
|
||||
### Simplification, Go Idioms, And Efficiency
|
||||
|
||||
Review long or branch-heavy functions for separable decisions, state
|
||||
transitions, or data transformations. Pay particular attention to orchestration,
|
||||
configuration validation, artifact dependency resolution, resume verification,
|
||||
restore/previous-cache planning, and analyze/publish selection logic. A useful
|
||||
refactor should reduce cognitive load while leaving the important ordering
|
||||
visible.
|
||||
|
||||
Check for:
|
||||
|
||||
- unnecessary nesting, defensive branches made unreachable by earlier
|
||||
validation, repeated normalization, and overly wide parameter lists;
|
||||
- interfaces defined for hypothetical extensibility rather than a demonstrated
|
||||
consumer boundary;
|
||||
- manual slice, map, string, error, and filesystem logic with a clearer standard
|
||||
library form;
|
||||
- incorrect or inconsistent `errors.Is`/`errors.As`, wrapping, context
|
||||
propagation, deferred cleanup, response-body closure, process waiting, and
|
||||
goroutine/channel ownership;
|
||||
- redundant filesystem scans, `stat`/checksum passes, whole-file buffering,
|
||||
copying, YAML/JSON round trips, sorting, remote listings, downloads, uploads,
|
||||
or adapter initialization;
|
||||
- linear searches nested in loops and repeated dependency or artifact lookup
|
||||
that should use an indexed map or a single planning pass;
|
||||
- unbounded concurrency, leaked work after cancellation, serialized independent
|
||||
work, and nondeterministic result collection; and
|
||||
- obsolete dependencies, portability assumptions, and platform-sensitive path
|
||||
or atomic-rename behavior.
|
||||
|
||||
Keep correctness and diagnosability ahead of micro-optimization. When a simpler
|
||||
algorithm changes performance characteristics, specify the representative
|
||||
input size and validation method.
|
||||
|
||||
### Comments And Local Explanation
|
||||
|
||||
Review high fan-in, high-complexity, security-sensitive, and commit-boundary
|
||||
code after likely simplifications have been identified. Add a comment
|
||||
recommendation when a maintainer needs to know why:
|
||||
|
||||
- state transitions or persistence operations occur in a specific order;
|
||||
- a stale record intentionally retains data while another transition clears it;
|
||||
- a path is checked more than once to resist traversal, symlink replacement, or
|
||||
time-of-check/time-of-use hazards;
|
||||
- an artifact is accepted only with particular manifest, checksum, contract, or
|
||||
provenance evidence;
|
||||
- a partial operation is intentionally not rolled back;
|
||||
- a remote pointer or local manifest must be installed last; or
|
||||
- concurrency, cancellation, compatibility, or downstream-tool behavior makes
|
||||
an apparently simpler approach unsafe.
|
||||
|
||||
Prefer a named helper, typed state, or smaller control flow when that removes the
|
||||
need for explanation. Check existing comments for stale claims as well as
|
||||
missing rationale.
|
||||
|
||||
### Test Suite Against The Canonical Policy
|
||||
|
||||
Build a risk-to-test matrix rather than auditing tests file by file in
|
||||
isolation. For each important behavior, identify its proper owner—parser,
|
||||
validator, domain package, adapter, orchestrator, CLI, integration, or end to
|
||||
end—and identify all tests that claim to protect it.
|
||||
|
||||
Evaluate:
|
||||
|
||||
- protection of data integrity, destructive operations, compatibility,
|
||||
security, concurrency, idempotency, recovery, and partial failure;
|
||||
- manifest transitions, force/invalidation, resume validation, atomic
|
||||
materialization, publish commit order, restore install order, and cleanup
|
||||
gates as assembled behaviors;
|
||||
- realistic HTTP, subprocess, filesystem, and object-store boundary behavior,
|
||||
including cancellation and malformed responses;
|
||||
- whether higher-level tests intentionally sample lower-level behavior or
|
||||
redundantly reproduce its full policy;
|
||||
- whether tests assert durable outcomes or private constants, exact error text,
|
||||
incidental paths, call choreography, or oversized snapshots;
|
||||
- whether real fast collaborators could replace elaborate doubles, and whether
|
||||
stateful fakes are realistic enough for the risk they protect;
|
||||
- fixture/helper duplication, oversized test cases, and setup that obscures the
|
||||
behavior under test without introducing a heavyweight test framework;
|
||||
- deterministic, offline, credential-free, order-independent execution and
|
||||
safe handling of environment and process-global state;
|
||||
- focused fuzz candidates in parsing, normalization, source IDs, remote/local
|
||||
path mapping, manifest decoding, and configuration boundaries; and
|
||||
- the presence and value of a small number of representative assembled
|
||||
workflows.
|
||||
|
||||
Use coverage only to locate unexpectedly weak consequential branches. Also
|
||||
inspect packages with extensive coverage for redundant tests and refactoring
|
||||
friction. For every proposed addition, deletion, or consolidation, state the
|
||||
realistic defect and marginal confidence involved.
|
||||
|
||||
The audit baseline should include the repository's canonical commands plus
|
||||
targeted diagnostic runs where supported:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./cmd/narratio
|
||||
```
|
||||
|
||||
Use focused repeated or shuffled runs to investigate state leakage and
|
||||
flakiness, and collect package/branch coverage for diagnosis. Review continuous
|
||||
integration to determine whether the appropriate offline validation is enforced;
|
||||
do not turn coverage percentage into a gate merely for this audit.
|
||||
|
||||
## Area-By-Area Inspection Map
|
||||
|
||||
| Area | Primary locations | What to inspect |
|
||||
| --- | --- | --- |
|
||||
| Process and application boundary | `cmd/narratio`, `internal/app` | Command dispatch, configuration selection, production composition, secret loading, lock lifetime, object-store initialization, context/error propagation, and separation of CLI reporting from orchestration policy. Review operator commands for consistent current-state authority and shared read-only mechanics. |
|
||||
| Stage registry and runner | `internal/stage/placeholders.go`, `internal/stage/stage.go`, `internal/app/planner.go`, `internal/app/runner.go`, `internal/app/run_stage.go` | Canonical order, action decisions, resume/force/self-skip/failure transitions, downstream invalidation, session/run manifest consistency, resource lifecycle, cleanup triggering, and opportunities to decompose the runner without hiding its state machine. |
|
||||
| Configuration | `internal/config` | Strict decoding, discovery and precedence, centralized defaults, normalization, templating, validation order, unknown fields, empty-value behavior, secret references, cross-field constraints, path confinement, deterministic errors, duplicated validator policy, and compatibility with maintained examples. |
|
||||
| Prepare and audio | `internal/stage/prepare.go`, `internal/audio`, `internal/previouscache` | Local/S3 exclusivity, cache and spool identity, partial downloads, checksum/reuse policy, previous-session required/optional planning, deterministic input records, clearing semantics, traversal safety, and avoiding repeated remote or filesystem work. |
|
||||
| Transcript stages | `internal/stage/transcribe.go`, `merge.go`, `polish.go`, `normalize.go`, `trim.go`, `render.go` | Contract parity across similar stages, bounded concurrency and cancellation, deterministic speaker/input ordering, run-local validation and canonical promotion, report/diagnostic classification, disabled behavior, and narrow opportunities for shared mechanics. |
|
||||
| Extraction | `internal/stage/extract.go`, `extract_resume.go`, `internal/adapters/notarius`, `internal/fileops/directory.go` | External receipt and lane validation, configuration fingerprint limits, immutable promotion, symlink/root replacement defenses, provenance and checksum checks, immediate and cross-invocation reuse, obsolete versus unsafe outcomes, failure residue, and whether dense verification logic can be clarified without weakening it. |
|
||||
| Analyze and artifact dependencies | `internal/stage/analyze.go`, `internal/artifacts`, `internal/artifactpolicy` | Source-family validation, runtime catalog state, enabled/selected/reused distinctions, topological ordering and cycle handling, required/optional inputs, local-only previous sources, deterministic metadata, repeated lookup/scanning, and ownership shared with config and publish. |
|
||||
| Publish and cleanup | `internal/stage/publish.go`, `internal/app/post_publish_cleanup.go`, `internal/app/cleanup_targets.go` | Prerequisite success, output selection, locks, required/optional behavior, exclusion rules, deterministic upload set, retry/idempotency implications, current-manifest then commit-marker ordering, metadata gates, and destructive path confinement. |
|
||||
| Manifest state | `internal/manifest` | Validation and backward compatibility, atomic persistence, timestamps, session/run identity, transition truth table, clearing versus retaining payload, create/load/save duplication, failure during dual-manifest updates, and whether state mutation has a single owner. |
|
||||
| Artifacts, paths, and policy | `internal/artifacts`, `internal/artifactpolicy`, `internal/pathsafe` | Canonical helper coverage, ad hoc reconstruction by callers, source-ID ownership, manifest-first resolution, extraction/current-state identity, destination normalization, stable ordering, typed missing-state errors, symlink/traversal defenses, and duplicate policy across config/stages/app. |
|
||||
| Restore | `internal/app/restore*.go`, `internal/previouscache`, `internal/audio` | Remote authority, confined mapping, deterministic plan actions, local conflict and force behavior, dry-run purity, temp-file installation, manifest-last ordering, partial failure/retry behavior, report accuracy, cache reuse, and shared current-state mechanics. |
|
||||
| File operations | `internal/fileops`, `internal/pathsafe`, local-store code in `internal/artifacts` | Atomic-write and promotion guarantees, permissions, close/sync/rename error handling, temp cleanup, same-filesystem assumptions, replacement policy, regular-file-only traversal, symlink and root-swap resistance, lock cleanup, and portability. |
|
||||
| External adapters and storage | `internal/adapters`, `internal/audio` | Transport isolation, shared subprocess mechanics versus adapter-specific policy, command/config duplication, quoting and working directories, timeouts/cancellation, stdout/stderr separation, HTTP body and retry behavior, S3 pagination/streaming/not-found mapping, credential independence, and external error adaptation. |
|
||||
| Shared models and diagnostics | `internal/artifactmodel`, `internal/contracts`, `internal/logging` | Serialization and validation invariants, unnecessary conversions, ownership of shared types, stable diagnostic structure, redaction, and whether small shared packages remain cohesive. |
|
||||
| Tests, examples, and automation | all `*_test.go`, `examples/`, `.woodpecker/` | Risk ownership, semantic duplication, fixture cost, policy-coupled assertions, realistic boundary tests, end-to-end sufficiency, default-suite isolation, example validation, diagnostic coverage, flakiness, runtime cost, and enforcement of canonical validation. |
|
||||
|
||||
## Narratio-Specific Cross-Boundary Scenarios
|
||||
|
||||
In addition to package-local review, trace these complete scenarios because a
|
||||
modular pipeline can look correct within every package while violating an
|
||||
end-to-end invariant:
|
||||
|
||||
1. A stage succeeds, its result becomes non-resumable, the rerun fails, and a
|
||||
later invocation decides what remains usable.
|
||||
2. An upstream forced or changed outcome interacts with already-succeeded,
|
||||
self-skipped, and disabled downstream stages.
|
||||
3. Extraction produces a valid immutable bundle, then configuration or
|
||||
transitive Notarius inputs change before analyze or publish.
|
||||
4. Previous-session state is published, restored or prepared into the local
|
||||
cache, and consumed by analyze without an unintended remote read.
|
||||
5. Publish fails at each upload boundary, especially between current manifest
|
||||
and current-run pointer, followed by status, restore, and retry.
|
||||
6. Restore encounters identical files, conflicting files, unsafe remote keys,
|
||||
cache hits, and a failure immediately before manifest installation.
|
||||
7. Automatic or manual cleanup is requested after skipped, failed, locked,
|
||||
partially uploaded, and fully committed publish outcomes.
|
||||
8. Cancellation reaches bounded transcription work, HTTP requests,
|
||||
subprocesses, object storage, and manifest reporting without leaks or false
|
||||
success.
|
||||
9. A configured artifact is disabled, unselected, reused, generated from
|
||||
another artifact, sourced from extraction, or sourced from a previous
|
||||
session, then filtered for publish.
|
||||
10. The same session is invoked concurrently, including lock contention and
|
||||
cleanup/release failures.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
The audit is complete when:
|
||||
|
||||
- every area in the inspection map has been reviewed against its canonical
|
||||
contracts and focused tests;
|
||||
- the stage lifecycle matrix and cross-boundary scenarios have explicit
|
||||
conclusions;
|
||||
- duplication candidates have been classified rather than merely counted;
|
||||
- simplification and performance recommendations explain their correctness
|
||||
constraints and expected benefit;
|
||||
- comment recommendations identify the non-obvious rationale to preserve;
|
||||
- the test suite has a risk-based sufficiency assessment, including gaps,
|
||||
redundancy, durability, execution properties, and automation;
|
||||
- findings are deduplicated, evidence-backed, and ranked by risk and dependency;
|
||||
and
|
||||
- unresolved questions and intentionally accepted risks are recorded rather
|
||||
than silently omitted.
|
||||
|
||||
## Execution
|
||||
|
||||
The [Audit Sequence](audit-sequence.md) is the canonical owner of execution
|
||||
order, stage boundaries, checkpoints, validation, and audit deliverables. This
|
||||
document remains the canonical owner of audit scope, review criteria, and the
|
||||
finding standard.
|
||||
734
docs/roadmap/audit-sequence.md
Normal file
734
docs/roadmap/audit-sequence.md
Normal file
@@ -0,0 +1,734 @@
|
||||
# Codebase Audit Sequence
|
||||
|
||||
Status: proposed
|
||||
|
||||
## Purpose And Relationship To The Audit Plan
|
||||
|
||||
This document turns the [Codebase Audit Plan](audit-plan.md) into a bounded,
|
||||
execution-ready sequence. The plan owns scope, review criteria, and the finding
|
||||
standard. This document owns ordering, dependencies, working records,
|
||||
validation, and exit gates.
|
||||
|
||||
The sequence is for investigation only. Do not mix production refactors or bug
|
||||
fixes into the audit. A confirmed urgent defect may justify stopping to request
|
||||
a separate remediation change, but its fix is not part of this sequence.
|
||||
|
||||
## Audit Run Records
|
||||
|
||||
Create `docs/roadmap/audit-findings.md` when the audit begins. It is the single
|
||||
working ledger and final audit report. Initialize it with:
|
||||
|
||||
- the audited revision, branch/worktree state, Go version, platform, and audit
|
||||
date;
|
||||
- baseline command results and timings;
|
||||
- an area coverage ledger;
|
||||
- the stage lifecycle matrix;
|
||||
- the cross-boundary scenario matrix from the audit plan;
|
||||
- a risk-to-test matrix;
|
||||
- candidate and confirmed finding registers; and
|
||||
- unresolved questions, accepted risks, and final conclusions.
|
||||
|
||||
Track each execution stage in the coverage ledger with one of `not_started`,
|
||||
`in_progress`, `complete`, or `blocked`. For a completed stage, record:
|
||||
|
||||
- contracts, packages, files, and important symbols reviewed;
|
||||
- graph traces, commands, tests, or other evidence used;
|
||||
- finding and candidate IDs produced;
|
||||
- explicit no-finding conclusions for reviewed high-risk behavior; and
|
||||
- follow-up questions assigned to later stages.
|
||||
|
||||
Use stable finding IDs with these prefixes:
|
||||
|
||||
| Prefix | Category |
|
||||
| --- | --- |
|
||||
| `COR` | Confirmed correctness defect |
|
||||
| `RSK` | Correctness or operational risk |
|
||||
| `ARC` | Ownership or architectural-boundary issue |
|
||||
| `DUP` | Duplicated mechanism or policy |
|
||||
| `SIM` | Simplification or idiomatic-Go opportunity |
|
||||
| `EFF` | Efficiency or resource-use issue |
|
||||
| `COM` | Missing, misleading, or stale explanatory comment |
|
||||
| `TST` | Test-suite gap, redundancy, brittleness, or execution issue |
|
||||
|
||||
Candidate IDs remain candidates until manual inspection confirms the behavior,
|
||||
contract, realistic scenario, and affected callers. Rejected candidates remain
|
||||
in a short classification log so later stages do not reopen them without new
|
||||
evidence.
|
||||
|
||||
## Execution Rules
|
||||
|
||||
1. Pin the audit to the revision recorded in Stage 0. If the worktree or HEAD
|
||||
changes, record the change and rerun every affected stage; do not silently
|
||||
combine evidence from different implementations.
|
||||
2. Use codebase graph search and call/data-flow traces before broad source
|
||||
search. Read the exact implementation, focused tests, and canonical contract
|
||||
before confirming a finding.
|
||||
3. Record test-policy observations during every behavior pass. Stage 12 owns the
|
||||
suite-wide conclusion but must not rediscover the suite from scratch.
|
||||
4. Record cross-area observations as candidates for the stage that owns the
|
||||
conclusion. Avoid producing duplicate findings from several review passes.
|
||||
5. Treat baseline failures as evidence, not automatic blockers. Continue when
|
||||
read-only inspection remains sound, and state the limitation. Stop only when
|
||||
the repository cannot be identified, required sources are unavailable, or a
|
||||
failure makes later evidence unreliable.
|
||||
6. Do not exercise a suspected destructive, credentialed, paid, or live-service
|
||||
path merely to prove a defect. Use source reasoning, existing safe fakes, or
|
||||
a narrowly controlled offline reproduction.
|
||||
7. Escalate a credible active data-loss, secret-exposure, or unsafe-cleanup
|
||||
defect immediately. Preserve the evidence and do not wait for final
|
||||
synthesis before reporting it.
|
||||
8. A stage is complete only when its exit gate is met. A package test passing is
|
||||
evidence, not proof that the review is complete.
|
||||
|
||||
## Sequence Overview
|
||||
|
||||
| Stage | Focus | Depends on | Primary result |
|
||||
| --- | --- | --- | --- |
|
||||
| 0 | Pin revision and establish baseline | None | Reproducible audit record |
|
||||
| 1 | Contract, boundary, and lifecycle map | 0 | Review matrices and ownership map |
|
||||
| 2 | Runner and manifest state machine | 1 | Lifecycle and dual-ledger conclusions |
|
||||
| 3 | Paths, artifacts, and filesystem safety | 1-2 | State/path authority and mutation conclusions |
|
||||
| 4 | Publish, remote commit, and cleanup | 2-3 | Commit-boundary and destructive-operation conclusions |
|
||||
| 5 | Restore and remote/previous state | 2-4 | Restore authority and recovery conclusions |
|
||||
| 6 | Configuration and application composition | 1-5 | Validation and wiring conclusions |
|
||||
| 7 | External adapters and shared support | 3, 6 | Boundary, cancellation, and resource conclusions |
|
||||
| 8 | Prepare and transcript-processing stages | 2-3, 6-7 | Ordinary stage-contract conclusions |
|
||||
| 9 | Extraction vertical slice | 2-3, 6-7 | Promotion, provenance, and resume conclusions |
|
||||
| 10 | Analyze and artifact dependency slice | 3, 6, 8-9 | Dependency and source-resolution conclusions |
|
||||
| 11 | Cross-codebase duplication, simplicity, efficiency, and comments | 2-10 | Classified maintainability candidates |
|
||||
| 12 | Test-suite policy audit | 2-11 | Risk-based suite sufficiency assessment |
|
||||
| 13 | Synthesis and audit closeout | 0-12 | Final deduplicated audit report |
|
||||
|
||||
Stages are intentionally ordered. Later stages may resolve candidates raised by
|
||||
earlier ones, but they must not invalidate an earlier stage silently. Return to
|
||||
the owning stage, update its coverage record, and note the new evidence.
|
||||
|
||||
## Stage 0: Pin Revision And Establish Baseline
|
||||
|
||||
### Entry
|
||||
|
||||
- Repository root and `docs/development.md` are available.
|
||||
- The audit plan and canonical policy documents can be read.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Record `git rev-parse HEAD`, branch/detached state, `git status --short`,
|
||||
`go version`, `go env GOOS GOARCH`, and the current date.
|
||||
2. Confirm that the code knowledge graph represents the recorded repository and
|
||||
revision; refresh the index if it is missing or stale.
|
||||
3. Capture the package/file/test inventory, entry points, architecture
|
||||
boundaries, high fan-in symbols, complexity signals, and similarity signals.
|
||||
4. Run the default offline baseline and record wall time and failures:
|
||||
|
||||
```sh
|
||||
go test -count=1 ./...
|
||||
go test -race -count=1 ./...
|
||||
go vet ./...
|
||||
```
|
||||
|
||||
5. Build into an external temporary directory so validation does not add a
|
||||
workspace binary:
|
||||
|
||||
```sh
|
||||
audit_build_dir="$(mktemp -d)"
|
||||
go build -o "$audit_build_dir/narratio" ./cmd/narratio
|
||||
go test -coverprofile="$audit_build_dir/coverage.out" ./...
|
||||
```
|
||||
|
||||
6. Inventory the repository's CI/release validation, maintained examples, fuzz
|
||||
tests, golden data, opt-in tests, and generated-test update mechanisms.
|
||||
|
||||
### Output
|
||||
|
||||
- Baseline and inventory sections in `audit-findings.md`.
|
||||
- Initial coverage ledger containing Stages 0-13.
|
||||
- Unconfirmed metric-driven candidates, clearly labeled as such.
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Revision and environment are reproducible.
|
||||
- Every baseline command has a recorded result.
|
||||
- Graph freshness is known.
|
||||
- Any limitation that affects later stages has an owner and disposition.
|
||||
|
||||
## Stage 1: Build The Contract, Boundary, And Lifecycle Map
|
||||
|
||||
### Entry
|
||||
|
||||
- Stage 0 is complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Read the architecture, internal overview, testing policy, focused internal
|
||||
documents, and the relevant CLI/configuration/operations/integration
|
||||
contracts using the development guide's routing rules.
|
||||
2. Map each package and important interface to its owned policy. Mark every
|
||||
cross-package dependency that appears to reverse or blur the intended
|
||||
direction for later confirmation.
|
||||
3. Build a stage-contract matrix with canonical order, declared inputs,
|
||||
outputs, configuration, adapters, skip behavior, resume validation,
|
||||
materialization boundary, manifest effects, and downstream invalidation.
|
||||
4. Build the lifecycle matrix required by the audit plan: first run,
|
||||
already-succeeded skip, self-skip, failure, interruption, forced replacement,
|
||||
non-resumable result, and successful rerun.
|
||||
5. Assign each of the ten cross-boundary scenarios in the audit plan to its
|
||||
primary execution stage and list supporting packages/tests.
|
||||
6. Seed the risk-to-test matrix with the intended test owner for each
|
||||
architectural invariant. Do not judge sufficiency yet.
|
||||
|
||||
### Output
|
||||
|
||||
- Package ownership, stage-contract, lifecycle, scenario, and preliminary
|
||||
risk-to-test matrices.
|
||||
- `ARC` and `RSK` candidates for apparent disagreements, without deciding from
|
||||
documentation alone which artifact is wrong.
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every area in the audit plan's inspection map has an assigned stage.
|
||||
- Every architectural invariant has an implementation owner and intended test
|
||||
owner.
|
||||
- Unknown or contradictory contracts are explicitly recorded.
|
||||
|
||||
## Stage 2: Audit The Runner And Manifest State Machine
|
||||
|
||||
### Entry
|
||||
|
||||
- Stage 1 matrices are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Trace the entry paths into full-run and single-stage execution through
|
||||
`internal/app/planner.go`, `runner.go`, `run_stage.go`, and related helpers.
|
||||
2. Inspect `internal/manifest` models, validation, session/run creation,
|
||||
loading, normalization, atomic saves, and all transition methods.
|
||||
3. Walk every lifecycle-matrix cell through both manifests. Verify clearing
|
||||
versus retention of outputs, diagnostics, generated configuration, metadata,
|
||||
errors, actions, timestamps, and downstream state.
|
||||
4. Reason about failures before and after each session-manifest and run-manifest
|
||||
save. Determine which disagreement states are possible and how a later
|
||||
invocation interprets them.
|
||||
5. Review force, changed-result, self-skip, failed-result, and non-resumable
|
||||
invalidation separately. Confirm behavior at the first and last canonical
|
||||
stage.
|
||||
6. Review session lock acquisition/release and concurrent invocation behavior,
|
||||
while leaving path implementation details to Stage 3.
|
||||
7. Classify the runner's complexity and repeated session/run persistence paths:
|
||||
state-machine clarity, justified explicitness, candidate local helpers, and
|
||||
comments that preserve ordering rationale.
|
||||
8. Review focused app/manifest tests against the matrix and add observations to
|
||||
the risk-to-test ledger.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/app ./internal/manifest
|
||||
go test -race -count=1 ./internal/app ./internal/manifest
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every lifecycle cell has a source-backed conclusion for both manifests.
|
||||
- Cross-boundary scenarios 1, 2, and the lock portion of 10 are resolved or
|
||||
carry explicit questions.
|
||||
- All runner/manifest candidates are confirmed, rejected, or assigned to a
|
||||
named later stage.
|
||||
|
||||
## Stage 3: Audit Paths, Artifacts, And Filesystem Safety
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 1-2 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Review `internal/artifacts`, `internal/artifactpolicy`, `internal/pathsafe`,
|
||||
`internal/fileops`, and local-store filesystem code.
|
||||
2. Inventory canonical path and key helpers, then search callers for ad hoc
|
||||
reconstruction, double normalization, mixed slash/filesystem semantics, or
|
||||
policy implemented outside its owner.
|
||||
3. Trace built-in, configured, extraction, previous-session, and current-state
|
||||
artifact resolution. Verify identity, checksum, contract, provenance,
|
||||
deterministic ordering, and typed missing-state behavior.
|
||||
4. Review atomic file writes, copies, directory promotion, temp cleanup,
|
||||
permission preservation, close/sync/rename errors, existing-destination
|
||||
behavior, same-filesystem assumptions, and platform sensitivity.
|
||||
5. Walk traversal, absolute path, broad root, symlink component, inspected-root
|
||||
replacement, non-regular file, and time-of-check/time-of-use scenarios.
|
||||
6. Confirm that low-level file/storage helpers receive explicit destinations
|
||||
and do not infer stage, campaign, session, run, or publish policy.
|
||||
7. Inspect lock-file implementation and cleanup errors to finish scenario 10.
|
||||
8. Record focused test ownership and gaps without duplicating Stage 2's state
|
||||
conclusions.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/artifacts ./internal/artifactpolicy ./internal/pathsafe ./internal/fileops
|
||||
go test -race -count=1 ./internal/artifacts ./internal/fileops
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every canonical path/key family has one identified owner.
|
||||
- Every material filesystem mutation has documented confinement and atomicity
|
||||
conclusions.
|
||||
- Scenario 10 is resolved.
|
||||
- Safety checks that appear repetitive are classified before any simplification
|
||||
recommendation is made.
|
||||
|
||||
## Stage 4: Audit Publish, Remote Commit, And Cleanup
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 2-3 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Trace publish from stage selection through object-store calls, manifest
|
||||
metadata, commit-marker publication, run completion, and post-publish
|
||||
cleanup.
|
||||
2. Verify prerequisite stage-state checks, selected/configured/extraction
|
||||
output resolution, required versus optional outputs, static and remote
|
||||
locks, run-file exclusions, previous-cache inclusion, and deterministic
|
||||
upload order.
|
||||
3. Enumerate failures before and after every upload. Prove that
|
||||
`current/run_id.txt` is written last and is the only remote-current commit
|
||||
point.
|
||||
4. Review retry/idempotency behavior, existing remote objects, partial uploads,
|
||||
pointer/manifest disagreement, and status/restore interpretation after each
|
||||
partial outcome.
|
||||
5. Trace automatic and manual cleanup gates. Confirm publish execution,
|
||||
`uploaded`, `current_pointer_written`, explicit policy, and confined targets
|
||||
are all required at the correct boundary.
|
||||
6. Confirm that `--force` cannot override publish locks or cleanup safety.
|
||||
7. Review duplication between publish planning, artifact destination policy,
|
||||
operator views, and cleanup metadata only after ownership is established.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/stage ./internal/app ./internal/artifacts ./internal/adapters/storage
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Cross-boundary scenarios 5 and 7 are resolved for every relevant failure
|
||||
boundary.
|
||||
- Remote-current authority and local-cleanup eligibility have explicit truth
|
||||
tables.
|
||||
- Publish findings distinguish stage policy from storage mechanics.
|
||||
|
||||
## Stage 5: Audit Restore And Remote/Previous State
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 2-4 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Trace restore discovery, planning, execution, reporting, audio
|
||||
materialization, and previous-cache planning through `internal/app`,
|
||||
`internal/artifacts`, `internal/previouscache`, `internal/audio`, and storage.
|
||||
2. Confirm remote pointer/manifest identity and campaign/session/run authority,
|
||||
including missing and inconsistent current state.
|
||||
3. Verify remote-to-local confinement, deterministic action ordering,
|
||||
`download`/`skip_same`/`conflict` decisions, force semantics, and dry-run
|
||||
purity.
|
||||
4. Walk failures during download, checksum or manifest validation, atomic
|
||||
install, report persistence, and the manifest-last boundary. Record the
|
||||
intentional lack of rollback and retry consequences.
|
||||
5. Review audio spool/cache identity, cache-hit verification, partial download
|
||||
behavior, and duplicate remote/filesystem work.
|
||||
6. Review previous-session requirement planning, required/optional behavior,
|
||||
identity checks, published-path fallback, and deterministic local mapping.
|
||||
7. Confirm which mechanics are shared with status/validate/operator commands
|
||||
and which caller-specific missing-state policies must remain separate.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/app ./internal/previouscache ./internal/audio ./internal/artifacts ./internal/adapters/storage
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Cross-boundary scenarios 4 and 6 are resolved through retry/recovery.
|
||||
- Restore authority, manifest-last installation, and partial-write behavior are
|
||||
explicit.
|
||||
- Previous-cache conclusions are ready for the prepare and analyze passes.
|
||||
|
||||
## Stage 6: Audit Configuration And Application Composition
|
||||
|
||||
### Entry
|
||||
|
||||
- Stage 1 is complete and Stages 2-5 have identified the policies that
|
||||
configuration and composition must supply.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Review `internal/config`, `cmd/narratio`, application command dispatch,
|
||||
configuration selection, secret-file environment loading, and production
|
||||
collaborator construction.
|
||||
2. Trace discovery, precedence, strict YAML decoding, defaults, empty values,
|
||||
normalization, session templating, and validation order across pipeline,
|
||||
campaign, and session configuration.
|
||||
3. Verify cross-field constraints for stage enablement, paths, timeouts,
|
||||
concurrency, artifacts, Notarius, Scriptorium, publish, storage, cleanup,
|
||||
audio, and previous-session behavior.
|
||||
4. Compare validation logic with maintained examples and the public
|
||||
configuration contract. Record contract drift rather than silently choosing
|
||||
code or docs.
|
||||
5. Check that filesystem secrets are loaded before the boundary that consumes
|
||||
them and are excluded from logs, manifests, reports, generated files, and
|
||||
errors.
|
||||
6. Review conditional construction of expensive/external collaborators and
|
||||
cleanup of anything with a lifecycle. Confirm test injection cannot create a
|
||||
behavior different from production composition.
|
||||
7. Classify repeated validators, path checks, timeout parsing, constructor
|
||||
wrappers, and single-stage command wrappers by policy owner.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/config ./internal/app ./cmd/narratio
|
||||
go vet ./...
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every operator-visible field used by audited behavior has a traced default,
|
||||
normalization, validation, and consumer.
|
||||
- Composition conclusions cover enabled and disabled stages without requiring
|
||||
live services or credentials.
|
||||
- Maintained examples have an explicit validity conclusion.
|
||||
|
||||
## Stage 7: Audit External Adapters And Shared Support
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 3 and 6 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Review `internal/adapters`, `internal/audio`, `internal/logging`,
|
||||
`internal/contracts`, and `internal/artifactmodel` at their public package
|
||||
boundaries.
|
||||
2. For each HTTP, subprocess, notification, and object-storage adapter, compare
|
||||
implementation with its integration contract and trace all production
|
||||
callers.
|
||||
3. Verify context cancellation, timeout ownership, process termination and
|
||||
waiting, goroutine/channel closure, HTTP response-body closure, retries,
|
||||
malformed responses, streaming, pagination, not-found mapping, and local
|
||||
file cleanup.
|
||||
4. Confirm command argument construction, working directory, environment,
|
||||
generated configuration, stdout/stderr separation, output validation, and
|
||||
external error adaptation stay inside the owning adapter.
|
||||
5. Compare subprocess implementations to the shared subprocess package. Classify
|
||||
repeated constructor/config/log/output mechanics separately from
|
||||
adapter-specific protocol policy.
|
||||
6. Review fakes for realistic state and concurrency behavior, but defer their
|
||||
suite-wide value judgment to Stage 12.
|
||||
7. Check shared models for avoidable conversions, stable serialization,
|
||||
validation ownership, and redaction-sensitive diagnostic fields.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/adapters/... ./internal/audio ./internal/logging ./internal/contracts ./internal/artifactmodel
|
||||
go test -race -count=1 ./internal/adapters/... ./internal/audio
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every external resource has an explicit acquisition, cancellation, and
|
||||
release conclusion.
|
||||
- Transport types and protocol policy have not leaked into stages.
|
||||
- Adapter duplication candidates identify the correct shared or specific
|
||||
owner.
|
||||
|
||||
## Stage 8: Audit Prepare And Transcript-Processing Stages
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 2-3 and 6-7 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Review `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and
|
||||
`render` as vertical slices from resolved configuration and manifest input
|
||||
through adapter call, run-local output, validation, canonical
|
||||
materialization, and recorded result.
|
||||
2. Verify each implementation against the Stage 1 contract matrix and focused
|
||||
internal document. Record any undeclared input, output, diagnostic, config,
|
||||
adapter, or skip/failure behavior.
|
||||
3. For prepare, confirm local/S3 exclusivity, stable input copying,
|
||||
previous-cache clearing/hydration, and deterministic manifest input records.
|
||||
4. For transcribe, confirm unique speaker identities, bounded runtime
|
||||
concurrency, cancellation, deterministic result ordering, adapter-returned
|
||||
path identity, and partial failure behavior.
|
||||
5. For transformation/render stages, confirm manifest-first resolution,
|
||||
run-local paths, schema/report validation, disabled/default behavior,
|
||||
canonical promotion, and diagnostic-versus-artifact classification.
|
||||
6. Compare similar stage implementations for shared mechanisms only after
|
||||
listing meaningful differences. Avoid a generic stage framework.
|
||||
7. Add stage-focused test ownership, gaps, and redundancy candidates to the
|
||||
risk-to-test matrix.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/stage ./internal/audio ./internal/previouscache ./internal/adapters/whisperx ./internal/adapters/seriatim ./internal/adapters/audita ./internal/adapters/scriptorium
|
||||
go test -race -count=1 ./internal/stage ./internal/audio
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every reviewed stage has a completed contract-matrix row.
|
||||
- Cross-boundary scenario 8 is resolved for transcription and subprocess-backed
|
||||
transformation stages.
|
||||
- Similarity candidates are classified as intentional explicitness, local
|
||||
helper candidates, or shared-owner findings.
|
||||
|
||||
## Stage 9: Audit The Extraction Vertical Slice
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 2-3 and 6-7 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Trace extraction from configuration validation and composition through
|
||||
transcript resolution, invocation fingerprint, Notarius execution, receipt
|
||||
and lane validation, directory promotion, manifest recording, catalog
|
||||
hydration, resume validation, analyze, and publish consumers.
|
||||
2. Verify run-local isolation, regular-file and confined-index requirements,
|
||||
required-lane policy, contract/provenance construction, checksum timing,
|
||||
immutable destination identity, and no-replacement promotion.
|
||||
3. Enumerate failures before and after subprocess completion, receipt parsing,
|
||||
payload inspection, promotion, and manifest persistence. Determine what
|
||||
remains diagnostic, durable, advertised, and reusable.
|
||||
4. Walk every resume validation branch. Distinguish obsolete/missing outcomes
|
||||
that trigger rerun from unsafe conditions that must stop execution.
|
||||
5. Evaluate the fingerprint's intentionally observable and unobservable inputs
|
||||
against documentation and force guidance.
|
||||
6. Review the dense validation code for named sub-decisions and comments while
|
||||
preserving the visible security proof and check ordering.
|
||||
7. Confirm focused tests cover immediate reuse, cross-invocation reuse,
|
||||
configuration change, payload tampering, provenance mismatch, symlinks/root
|
||||
replacement, failure residue, and downstream invalidation at the correct
|
||||
layers.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/stage ./internal/artifacts ./internal/fileops ./internal/adapters/notarius ./internal/app
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Cross-boundary scenario 3 is resolved, including transitive-input limits.
|
||||
- Promotion, advertisement, and resume each have a distinct authority and
|
||||
failure conclusion.
|
||||
- Every proposed simplification states which security or compatibility checks
|
||||
it preserves.
|
||||
|
||||
## Stage 10: Audit Analyze And Artifact Dependencies
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 3, 6, 8, and 9 are complete.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Trace all analyze source families from configuration validation through
|
||||
runtime catalog registration, availability, resolution, Scriptorium
|
||||
execution/reuse, materialization, metadata, and publish selection.
|
||||
2. Verify enabled, selected, executable, reused, generated, and unavailable
|
||||
states are distinct and deterministic.
|
||||
3. Review configured-artifact dependency validation and runtime topological
|
||||
ordering for cycles, missing dependencies, stable ordering, and consistency
|
||||
between configuration and execution.
|
||||
4. Confirm required/optional behavior and guidance for built-in transcripts,
|
||||
prepared stable inputs, configured artifacts, extraction sources, and
|
||||
previous-session sources.
|
||||
5. Prove previous-session resolution is local-only during analyze and that
|
||||
disabled artifacts are reused only under the documented conditions.
|
||||
6. Inspect repeated resolution branches, parameter width, nested lookup, and
|
||||
ordering work for a smaller representation or indexed plan without merging
|
||||
distinct source policies.
|
||||
7. Review tests for each state transition and source family at the narrowest
|
||||
stable owner, noting semantic duplication across config, artifacts, stage,
|
||||
publish, and assembled runner tests.
|
||||
|
||||
### Validation
|
||||
|
||||
```sh
|
||||
go test -count=1 ./internal/stage ./internal/artifacts ./internal/artifactpolicy ./internal/config ./internal/adapters/scriptorium ./internal/app
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Cross-boundary scenario 9 is resolved for every source family and selection
|
||||
state.
|
||||
- Dependency ordering and source availability have explicit determinism and
|
||||
complexity conclusions.
|
||||
- Config, artifact-policy, catalog, stage, and publish ownership is unambiguous
|
||||
or represented by an `ARC` finding.
|
||||
|
||||
## Stage 11: Audit Duplication, Simplicity, Efficiency, And Comments
|
||||
|
||||
### Entry
|
||||
|
||||
- Behavior stages 2-10 are complete, so structural candidates can be judged
|
||||
against known contracts.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Rerun graph similarity, complexity, fan-in/fan-out, call-path, loop-depth,
|
||||
scan-in-loop, and change-coupling analyses on production code. Add targeted
|
||||
text/static searches for patterns the graph cannot represent.
|
||||
2. Revisit all `DUP`, `SIM`, `EFF`, and `COM` candidates collected earlier.
|
||||
Search for additional occurrences and trace all callers before assigning an
|
||||
owner.
|
||||
3. For duplication, classify coincidental syntax, shared mechanism, duplicated
|
||||
policy, or deliberately explicit security/state logic. Propose only the
|
||||
narrowest helper that improves ownership and comprehension.
|
||||
4. For complexity, sketch the smaller control flow or data model and verify it
|
||||
leaves state transitions, validation order, and commit boundaries visible.
|
||||
5. For efficiency, state the input scale or call frequency, current and proposed
|
||||
complexity/I/O behavior, expected benefit, and benchmark or measurement
|
||||
needed. Reject micro-optimizations without a credible workload.
|
||||
6. Review standard-library usage, errors, slices/maps, allocations, copying,
|
||||
sorting, serialization, filesystem passes, adapter initialization, remote
|
||||
calls, goroutines/channels, and interface breadth across the complete codebase.
|
||||
7. Review comments only after simplification decisions. Recommend why-comments
|
||||
for remaining invariants, compatibility limits, safety checks, partial
|
||||
failure, and ordering; flag comments that restate code or no longer match it.
|
||||
8. Check dependencies and platform assumptions for clear correctness,
|
||||
portability, complexity, or maintenance consequences.
|
||||
|
||||
### Validation
|
||||
|
||||
- Run focused package tests for any behavior used to disprove or confirm a
|
||||
candidate.
|
||||
- Run existing benchmarks where relevant. Propose a benchmark rather than
|
||||
inventing performance claims when representative measurement is absent.
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every structural candidate is confirmed, rejected with a reason, or merged
|
||||
into a stronger root-cause finding.
|
||||
- No helper recommendation creates a generic workflow abstraction or moves
|
||||
policy into a low-level utility.
|
||||
- Every efficiency finding has a credible workload and validation method.
|
||||
- Every comment finding states the non-obvious rationale that should be
|
||||
preserved.
|
||||
|
||||
## Stage 12: Audit The Test Suite Against Policy
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 2-11 have populated the risk-to-test matrix and test observations.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Complete the risk-to-test matrix. For every consequential invariant, list
|
||||
the current tests, proper owner, protected defect, missing failure modes, and
|
||||
overlap with other layers.
|
||||
2. Review tests by behavior cluster rather than filename: parsing/validation,
|
||||
domain/state, filesystem, adapters, orchestration, CLI, integration, and
|
||||
representative assembled workflows.
|
||||
3. Classify gaps for data integrity, destructive operations, compatibility,
|
||||
security, concurrency, idempotency, recovery, cancellation, and partial
|
||||
success. Confirm the gap is not credibly protected elsewhere.
|
||||
4. Classify redundancy and brittleness: private constants/defaults, exact error
|
||||
wording, incidental formatting/paths, mock choreography, oversized
|
||||
snapshots, helper-level duplication, and the same policy repeated across
|
||||
layers.
|
||||
5. Review doubles using the policy order: real deterministic collaborator,
|
||||
stateful fake, stub, then mock when interaction is contractual. Check that
|
||||
fakes model the failure and state semantics used by the tests.
|
||||
6. Inspect test helpers and large test functions for simplification and
|
||||
meaningful table-driven boundaries without creating a fixture framework
|
||||
whose maintenance cost exceeds its value.
|
||||
7. Review determinism and isolation: credentials, network access, paid APIs,
|
||||
environment, working directory, clocks, randomness, ports, temp paths,
|
||||
process-global state, ordering, cleanup, and parallel execution.
|
||||
8. Use coverage to investigate consequential weak branches, not as a score.
|
||||
Review heavily covered behavior for marginal-value duplication as well.
|
||||
9. Identify focused fuzz opportunities for parsers, YAML/JSON normalization,
|
||||
source IDs, confined paths, remote/local mapping, and manifest decoding.
|
||||
10. Compare local requirements with `.woodpecker/` and other automation. Record
|
||||
missing enforcement as a risk/cost decision, not an assumption that every
|
||||
diagnostic command belongs in CI.
|
||||
11. Investigate order dependence and flakiness with bounded runs, recording
|
||||
runtime and any reproducible seed:
|
||||
|
||||
```sh
|
||||
go test -shuffle=on -count=3 ./...
|
||||
go test -race -shuffle=on -count=1 ./...
|
||||
```
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every important risk has a sufficiency conclusion and one intended test
|
||||
owner.
|
||||
- Every proposed test addition names the realistic defect and marginal value.
|
||||
- Every deletion/consolidation names the stronger remaining protection.
|
||||
- Default-suite determinism, offline behavior, runtime, flakiness, and CI
|
||||
enforcement have explicit conclusions.
|
||||
|
||||
## Stage 13: Synthesize And Close The Audit
|
||||
|
||||
### Entry
|
||||
|
||||
- Stages 0-12 meet their exit gates or have explicitly accepted limitations.
|
||||
|
||||
### Execute
|
||||
|
||||
1. Reconcile candidates and findings across stages. Merge shared root causes and
|
||||
remove repeated symptoms while retaining all affected locations and
|
||||
contracts.
|
||||
2. Recheck every confirmed finding against current source, callers, tests, and
|
||||
canonical documentation. Downgrade or reject anything supported only by a
|
||||
metric or hypothetical preference.
|
||||
3. Rank impact, likelihood, confidence, and remediation scope separately. Order
|
||||
the recommended backlog by dependency: correctness/data safety first,
|
||||
architectural ownership next, then simplification/duplication, tests,
|
||||
efficiency, and comments where they remain necessary.
|
||||
4. Record positive conclusions for high-risk areas where the current design and
|
||||
tests are sufficient. The report should not imply that only defective areas
|
||||
were reviewed.
|
||||
5. Reconcile the area coverage ledger, lifecycle matrix, cross-boundary scenario
|
||||
matrix, and risk-to-test matrix with the audit plan's completion criteria.
|
||||
6. Record any accepted risks, ambiguous contracts, environmental limitations,
|
||||
and deferred investigations with an explicit rationale and owner.
|
||||
7. Check whether HEAD or the worktree changed since Stage 0. Rerun affected
|
||||
stages or clearly pin the report to the original revision.
|
||||
8. Validate the report and roadmap document links and run `git diff --check`.
|
||||
If implementation changed during the audit, rerun the full Stage 0 validation
|
||||
baseline against the final audited revision.
|
||||
|
||||
### Final Deliverable
|
||||
|
||||
`docs/roadmap/audit-findings.md` must contain:
|
||||
|
||||
- an executive assessment without unsupported quality scores;
|
||||
- the audited revision and validation baseline;
|
||||
- coverage and scenario completion summaries;
|
||||
- confirmed findings ordered by dependency and risk;
|
||||
- rejected candidate themes where their recurrence would otherwise waste work;
|
||||
- the test-suite sufficiency assessment;
|
||||
- positive conclusions and accepted risks; and
|
||||
- a recommended remediation order, without implementing the remediation.
|
||||
|
||||
### Exit Gate
|
||||
|
||||
- Every completion criterion in the audit plan is satisfied or explicitly
|
||||
marked limited with rationale.
|
||||
- Every finding is evidence-backed, deduplicated, actionable, and assigned a
|
||||
stable ID.
|
||||
- No production change is included in the audit output.
|
||||
- The report is sufficient to prepare a separate remediation sequence without
|
||||
repeating discovery.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,35 +0,0 @@
|
||||
# S3 Archive Foundations Runbook
|
||||
|
||||
This runbook documents the currently implemented storage/archive foundations and the boundaries of current behavior.
|
||||
|
||||
## Implemented Now
|
||||
|
||||
- config modeling for:
|
||||
- `pipeline.storage.s3`
|
||||
- `pipeline.spool`
|
||||
- `pipeline.archive`
|
||||
- `session.inputs.audio_s3`
|
||||
- promotion rule validation for safe relative paths
|
||||
- run ID generation and path/key helper functions
|
||||
- manifest run/path identity fields
|
||||
- remote storage backend layer:
|
||||
- object-store interface (`List`, `Download`, `Upload`, `Exists`)
|
||||
- fake backend for deterministic tests
|
||||
- S3-compatible backend using AWS SDK v2
|
||||
- config-based backend construction helper
|
||||
|
||||
## Not Implemented Yet
|
||||
|
||||
- prepare-stage S3 object listing or download
|
||||
- archive-stage S3 upload or promotion writes
|
||||
- writing `current/manifest.json` or `current/run_id.txt` in S3
|
||||
|
||||
## Operational Notes
|
||||
|
||||
- local audio workflows remain the active development path (`audio_dir` or `audio_files`)
|
||||
- `audio_s3` and local audio config are mutually exclusive
|
||||
- do not place AWS credentials in Narratio config files
|
||||
|
||||
## Next Implementation Target
|
||||
|
||||
Use the storage backend layer in prepare-stage session audio discovery/download flow, while preserving local audio input support.
|
||||
@@ -1,84 +0,0 @@
|
||||
# S3 Audio Input
|
||||
|
||||
This document describes implemented S3 audio input behavior in `prepare`.
|
||||
|
||||
## Scope
|
||||
|
||||
Implemented:
|
||||
|
||||
- `prepare` can acquire source audio from S3 when `session.inputs.audio_s3.prefix` is configured.
|
||||
- object listing and download go through the storage backend abstraction.
|
||||
- tests use fake storage; no live S3 service is required for test runs.
|
||||
|
||||
Not implemented:
|
||||
|
||||
- uploads of failed runs
|
||||
|
||||
## Required Configuration
|
||||
|
||||
`pipeline.yml`:
|
||||
|
||||
- `storage.s3.bucket` must be set when S3 audio input is used.
|
||||
- `storage.s3.root_prefix` defaults to `dnd`.
|
||||
- `storage.s3.access_key_id_env` defaults to `OBJECT_STORAGE_KEY_ID`.
|
||||
- `storage.s3.secret_access_key_env` defaults to `OBJECT_STORAGE_KEY`.
|
||||
- `spool.root` defaults to `/var/spool/narratio`.
|
||||
|
||||
`session.yml`:
|
||||
|
||||
- configure `session.campaign` and `session.session_id`.
|
||||
- configure `session.inputs.audio_s3.prefix` for S3 audio input.
|
||||
- do not configure `inputs.audio_dir` or `inputs.audio_files` at the same time as `inputs.audio_s3`.
|
||||
|
||||
## Prefix Shape
|
||||
|
||||
Session S3 root:
|
||||
|
||||
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||
|
||||
Audio prefix:
|
||||
|
||||
`{session_root}/{audio_s3.prefix}`
|
||||
|
||||
Example:
|
||||
|
||||
`dnd/campaigns/forsaken/sessions/2026-04-19/audio/`
|
||||
|
||||
Audio files must already exist in S3 before running Narratio.
|
||||
|
||||
## Prepare Behavior
|
||||
|
||||
When `inputs.audio_s3.prefix` is configured, `prepare`:
|
||||
|
||||
1. lists objects under the computed S3 audio prefix
|
||||
2. filters to `.flac` objects
|
||||
3. fails when no `.flac` objects are found
|
||||
4. downloads selected objects to spool audio:
|
||||
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||
5. materializes audio into workdir audio:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
|
||||
6. records input provenance in the manifest (bucket, key, metadata, local paths, checksum)
|
||||
|
||||
Notes:
|
||||
|
||||
- `.flac` filtering is case-insensitive.
|
||||
- ETag is recorded as provider metadata only and is not treated as a checksum.
|
||||
|
||||
## Local Audio Development
|
||||
|
||||
Local audio workflows remain supported:
|
||||
|
||||
- `inputs.audio_dir`
|
||||
- `inputs.audio_files`
|
||||
|
||||
These options are mutually exclusive with `inputs.audio_s3`.
|
||||
|
||||
## Archive Boundary
|
||||
|
||||
Current archive behavior relevant to S3 audio input:
|
||||
|
||||
- successful runs are uploaded by archive under `runs/{run_id}/`
|
||||
- configured promotions are uploaded to session-level destinations
|
||||
- `current/manifest.json` and `current/run_id.txt` are published
|
||||
- local source audio is not re-uploaded by default
|
||||
- failed or incomplete runs are not uploaded
|
||||
@@ -1,62 +0,0 @@
|
||||
# Storage Backends
|
||||
|
||||
This document describes the currently implemented remote object storage backend layer used by Narratio, and its intended role in later prepare/archive work.
|
||||
|
||||
## Implemented
|
||||
|
||||
Remote object store abstraction:
|
||||
|
||||
- `List(ctx, prefix)`
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Object metadata model includes:
|
||||
|
||||
- key
|
||||
- size
|
||||
- ETag (provider metadata only)
|
||||
- last modified time when available
|
||||
|
||||
Backends:
|
||||
|
||||
- fake storage backend for deterministic tests
|
||||
- S3-compatible backend implemented with AWS SDK for Go v2
|
||||
|
||||
Construction:
|
||||
|
||||
- config-based constructor builds S3 backend from `pipeline.storage.s3` values:
|
||||
- bucket
|
||||
- region
|
||||
- endpoint
|
||||
- force_path_style
|
||||
- access_key_id_env
|
||||
- secret_access_key_env
|
||||
|
||||
## Key Invariant
|
||||
|
||||
- callers pass full bucket-relative object keys
|
||||
- storage backends do not prepend `root_prefix`
|
||||
- storage backends do not infer campaign/session/run paths
|
||||
|
||||
S3 session/run key builders remain separate and continue to live outside backend implementations.
|
||||
|
||||
## Security Boundary
|
||||
|
||||
- do not store AWS credentials in Narratio config
|
||||
- Narratio first checks configured env-var names (`access_key_id_env`, `secret_access_key_env`);
|
||||
when both are present and non-empty, it uses static credentials from those values
|
||||
- when either configured credential value is missing, Narratio falls back to the standard AWS SDK credential chain
|
||||
- AWS SDK-specific types remain isolated to the storage adapter package
|
||||
|
||||
## Testing
|
||||
|
||||
- fake storage tests cover list/download/upload/exists and error paths
|
||||
- S3 backend tests use injected fake S3 API clients
|
||||
- tests do not require live S3 services, AWS credentials, or network access
|
||||
|
||||
## Not Implemented Yet
|
||||
|
||||
- prepare-stage S3 object listing or downloads
|
||||
- archive-stage S3 uploads or promotion writes
|
||||
- writing `current/manifest.json` or `current/run_id.txt` to S3
|
||||
479
docs/troubleshooting.md
Normal file
479
docs/troubleshooting.md
Normal file
@@ -0,0 +1,479 @@
|
||||
# Troubleshooting
|
||||
|
||||
Operational diagnosis guide for common Narratio failures.
|
||||
|
||||
## Config file not found
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails to resolve `pipeline.yml`, `campaign.yml`, or `session.yml`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing files in default search paths;
|
||||
- wrong campaign selection;
|
||||
- omitted explicit flags.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||
|
||||
Relevant reference: [Configuration discovery](./config.md#discovery-and-selection).
|
||||
|
||||
## Session template placeholders rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- load error says session file must be concrete or contains `{{ ... }}` placeholders.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- using template content as runtime session config.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04 --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- generate concrete session YAML with `narratio session init`.
|
||||
|
||||
Relevant reference: [Operations: Session Initialization](./operations.md#session-initialization).
|
||||
|
||||
## Strict decode or schema validation failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown field / invalid value error during config load.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- stale field name, typo, invalid enum, or invalid duration/path format.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- align config with [Configuration](./config.md) and the
|
||||
[maintained examples](../examples/README.md).
|
||||
|
||||
Relevant reference: [Configuration](./config.md).
|
||||
|
||||
## Audio mode conflict
|
||||
|
||||
Symptom:
|
||||
|
||||
- validation fails on session audio configuration.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- configured both local and S3 session audio inputs.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
Relevant reference: [Session configuration](./config.md#session).
|
||||
|
||||
## `--artifacts` selection error
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown artifact key or invalid `--artifacts` usage.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- key not defined in `pipeline.scriptorium.artifacts`;
|
||||
- empty list entry (for example trailing comma);
|
||||
- `run-stage` used with non-`analyze`/`publish` target.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- provide only configured keys and use `--artifacts` with supported commands/stages.
|
||||
|
||||
Relevant reference: [CLI artifact selection](./cli.md).
|
||||
|
||||
## Notarius executable missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails while resolving or starting the Notarius executable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||
- a configured executable path is wrong.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- install a compatible Notarius release or correct the binary setting, then
|
||||
rerun extraction.
|
||||
|
||||
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius exits nonzero
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports a Notarius exit error instead of a receipt.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||
the receipt and is not merged with diagnostics.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||
|
||||
After a failed replacement, an older immutable bundle may still exist even
|
||||
though the current session manifest has no successful extraction payload. This
|
||||
is expected audit state, not a signal to relink the old bundle manually.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Atomic Notarius promotion unsupported
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||
before a durable bundle or temporary promotion tree is created.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio is running on an operating system other than Linux, macOS, or
|
||||
Windows, where the required atomic no-replace directory primitive has not
|
||||
been implemented and verified.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||
with a manual copy or move; the session manifest must never observe a partial
|
||||
or overwritten bundle.
|
||||
|
||||
This is an extraction-specific platform boundary, not a support statement for
|
||||
unrelated Narratio workflows. See
|
||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||
lane descriptor, or payload path even though Notarius exited successfully.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- Narratio and Notarius versions disagree on their consumer contract;
|
||||
- the configured pipeline or lane constraints are stale;
|
||||
- output paths escape the bundle or traverse symlinks.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- compare installed Notarius output with the canonical Notarius contracts,
|
||||
including receipt `index_file: index.json` and index management names
|
||||
`manifest.json`, `rejected.json`, and `warnings.json`; align
|
||||
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||
checks.
|
||||
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Required Notarius lane rejected or missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||
or incompatible, including after a zero exit.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||
- correct the Notarius module or the exact declared lane contract;
|
||||
- remove an output declaration only if downstream consumers genuinely no longer
|
||||
require that source, then rerun extraction.
|
||||
|
||||
Every configured output is required. Narratio does not promote a partial result.
|
||||
|
||||
## Extraction resume invalidated
|
||||
|
||||
Symptom:
|
||||
|
||||
- a previously successful extraction runs again during ordinary continuation.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||
configured output contracts changed;
|
||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||
checksum no longer validates.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||
investigate it rather than replacing files manually.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
Symptom:
|
||||
|
||||
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||
but Narratio still considers the previous extraction resumable.
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||
Notarius inputs. Always force extraction after changing them; downstream
|
||||
successful stages are then marked stale normally.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Previous-session artifact input missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare/analyze fails due to missing required previous-session artifact cache input.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing `session.previous_session_id`;
|
||||
- previous artifact not restored/published for source session.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
or rerun prepare after correcting session config:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Session lock conflict (`.lock`)
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails acquiring session lock.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- another process is running for the same session;
|
||||
- stale lock left by interrupted process.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
||||
ps aux | grep narratio
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- wait for active process completion;
|
||||
- remove stale lock only after confirming no live process owns it.
|
||||
|
||||
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
|
||||
|
||||
## Restore conflict without `--force`
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore fails with conflict count.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- local durable files differ from remote restore sources.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite local.
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Restore current-state discovery failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore cannot find current pointer or current manifest.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- no committed publish current state;
|
||||
- storage credentials or connectivity failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- resolve storage/auth issue;
|
||||
- republish from healthy local state if current pointer is missing.
|
||||
|
||||
Relevant reference: [Operations: Publish Workflow](./operations.md#publish-workflow).
|
||||
|
||||
## Publish output failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- publish fails on missing required source, upload error, or commit write.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- required source file not produced;
|
||||
- lock/state expectations mismatch;
|
||||
- remote storage failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04 --remote
|
||||
narratio session status 2026-04-04
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- regenerate missing sources by rerunning prerequisite stages;
|
||||
- correct publish source/destination rules;
|
||||
- retry after storage failure is resolved.
|
||||
|
||||
Relevant reference: [Publish configuration](./config.md#publish-configuration-summary).
|
||||
|
||||
## Render markdown source missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze or publish fails because `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` is unavailable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- render stage was not executed after transcript changes;
|
||||
- render stage failed before producing canonical markdown outputs.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- rerun render and then retry downstream stage(s):
|
||||
|
||||
```bash
|
||||
narratio run-stage render 2026-04-04 --force
|
||||
narratio run-stage analyze 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Stage Execution](./operations.md#stage-execution-and-continuation-behavior).
|
||||
|
||||
## Secrets or storage credential failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- object-store command fails at initialization/auth.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- invalid `pipeline.secrets.env_dir`;
|
||||
- missing credential environment variables;
|
||||
- invalid S3 endpoint/bucket settings.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -la /path/to/secrets_dir
|
||||
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct secret-file path and permissions;
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
Relevant reference: [Secrets](./config.md#secrets-handling).
|
||||
|
||||
## S3 audio prepare failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare fails listing/downloading session S3 audio.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- incorrect `session.inputs.audio_s3.prefix`;
|
||||
- no matching `.flac` objects;
|
||||
- storage connectivity or permissions failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- verify prefix contents and storage access;
|
||||
- keep session audio mode consistent.
|
||||
|
||||
Relevant reference: [Operations](./operations.md).
|
||||
|
||||
## References
|
||||
|
||||
- [docs/cli.md](./cli.md)
|
||||
- [docs/config.md](./config.md)
|
||||
- [docs/operations.md](./operations.md)
|
||||
- [docs/internal/stage-publish.md](./internal/stage-publish.md)
|
||||
48
examples/README.md
Normal file
48
examples/README.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Maintained Examples
|
||||
|
||||
These files are safe, copyable starting points for Narratio configuration and
|
||||
input structure. Replace placeholder identifiers, storage names, integration
|
||||
URLs, and paths for the target environment. Field meanings and defaults belong
|
||||
in the [configuration reference](../docs/config.md).
|
||||
|
||||
## Pipeline Configuration
|
||||
|
||||
- [Minimal pipeline](pipeline.minimal.yml): campaign discovery plus the required
|
||||
WhisperX URL.
|
||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||
|
||||
The existing `internal/config` example test loads and validates each pipeline
|
||||
with the sample campaign and a compatible local- or S3-audio session.
|
||||
|
||||
## Campaign And Session Configuration
|
||||
|
||||
- [Sample campaign](campaigns/sample-campaign/campaign.yml), its
|
||||
[session template](campaigns/sample-campaign/session.template.yml), and its
|
||||
adjacent stable inputs provide a complete campaign directory shape.
|
||||
- [Local-audio session](session.local-audio.yml) and
|
||||
[S3-audio session](session.s3-audio.yml) are concrete session files.
|
||||
- [Session template](session.template.yml) and the campaign-local equivalent
|
||||
demonstrate the narrow placeholder syntax consumed by `session init`; they
|
||||
are templates, not runtime session files.
|
||||
|
||||
## Input Fixtures
|
||||
|
||||
- [Speakers](speakers.yml), [autocorrect](autocorrect.yml), and
|
||||
[glossary](glossary.yml) show the standalone input shapes.
|
||||
- The sample campaign references its local
|
||||
[speakers](campaigns/sample-campaign/speakers.yml),
|
||||
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
|
||||
[glossary](campaigns/sample-campaign/glossary.yml),
|
||||
[players](campaigns/sample-campaign/players.yml), and
|
||||
[party](campaigns/sample-campaign/party.yml) fixtures.
|
||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
The examples contain environment-variable names but no credential values. They
|
||||
use fictional campaign content and reserved example domains.
|
||||
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
campaign_id: sample-campaign
|
||||
session_template_file: ./session.template.yml
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
2
examples/campaigns/sample-campaign/party.yml
Normal file
2
examples/campaigns/sample-campaign/party.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
- name: Example Hero
|
||||
type: pc
|
||||
2
examples/campaigns/sample-campaign/players.yml
Normal file
2
examples/campaigns/sample-campaign/players.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
- name: Example Player
|
||||
role: player
|
||||
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
match:
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
@@ -1,26 +0,0 @@
|
||||
workspace:
|
||||
root: ./tmp/narratio-workspace
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
seriatim:
|
||||
binary: "seriatim"
|
||||
timeout: "10m"
|
||||
output_schema: "seriatim-intermediate"
|
||||
coalesce_gap: 3.0
|
||||
|
||||
audita:
|
||||
binary: "audita"
|
||||
timeout: "3h"
|
||||
base_url: "https://openrouter.ai/api/v1"
|
||||
model: "openrouter/google/gemma-4-31b-it"
|
||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||
modules: ["glossary", "homophones", "spoken_word", "grammar"]
|
||||
output_schema: "audita-v1"
|
||||
work_dir_retention: "auto"
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_model: "openrouter/google/gemma-4-31b-it"
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||
# only the three Notarius lanes it needs.
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
notarius:
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
artifacts:
|
||||
session_brief:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_brief
|
||||
output_path: artifacts/session_brief.md
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.npc_registry
|
||||
required: true
|
||||
locations:
|
||||
source: narratio.extraction.location_registry
|
||||
required: true
|
||||
scenes:
|
||||
source: narratio.extraction.scene_descriptions
|
||||
required: true
|
||||
|
||||
266
examples/pipeline.full.annotated.yml
Normal file
266
examples/pipeline.full.annotated.yml
Normal file
@@ -0,0 +1,266 @@
|
||||
# Full annotated pipeline example for implemented Narratio config fields.
|
||||
# Values are safe placeholders and must be adapted per environment.
|
||||
|
||||
workspace:
|
||||
# Optional: defaults to /var/lib/narratio.
|
||||
root: /var/lib/narratio/workspace
|
||||
# Optional: remove run-scoped workdir after successful publish commit.
|
||||
cleanup_after_publish: false
|
||||
|
||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||
# secrets:
|
||||
# env_dir: ./secrets
|
||||
|
||||
storage:
|
||||
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
||||
backend: s3
|
||||
s3:
|
||||
# Required when using S3 audio or S3 publish uploads.
|
||||
bucket: my-dnd-archive
|
||||
# Optional; defaults to "dnd".
|
||||
root_prefix: dnd
|
||||
# Optional region/endpoint settings.
|
||||
region: us-east-1
|
||||
endpoint: ""
|
||||
force_path_style: false
|
||||
# Optional; defaults shown explicitly.
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
# Optional command default when --campaign is omitted.
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
# Optional; defaults to /var/spool/narratio.
|
||||
root: /var/spool/narratio
|
||||
# Optional cleanup of run-scoped spool audio after successful publish commit.
|
||||
delete_audio_after_publish: false
|
||||
|
||||
publish:
|
||||
# Optional booleans; defaults are true.
|
||||
enabled: true
|
||||
upload_run: true
|
||||
# Optional publish output rules; sources use Narratio artifact source IDs.
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||
# are never implicit publish sources.
|
||||
- source: narratio.extraction.npc_registry
|
||||
dest: artifacts/extraction/npc-registry.json
|
||||
required: true
|
||||
|
||||
whisperx:
|
||||
# Required.
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
language: en
|
||||
timeout: 30m
|
||||
retries: 3
|
||||
retry_delay: 2s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
env:
|
||||
# Optional advanced tuning; set only when needed.
|
||||
overlap_word_run_gap: 1.0
|
||||
overlap_word_run_reorder_window: 1.0
|
||||
backchannel_max_duration: 2.0
|
||||
filler_max_duration: 1.25
|
||||
|
||||
audita:
|
||||
# Optional overrides; defaults shown explicitly where applicable.
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
base_url: ""
|
||||
model: ""
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_model: ""
|
||||
validation_llm_concurrency: 1
|
||||
transcript_description: ""
|
||||
config_path: /usr/local/etc/audita/config.yml
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
# Optional; defaults shown explicitly.
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
trim:
|
||||
# Optional; defaults shown explicitly.
|
||||
enabled: true
|
||||
output_path: transcripts/final.trimmed.json
|
||||
bounds:
|
||||
prompt_id: dnd.session_bounds
|
||||
profile_id: ""
|
||||
transcript_input_name: transcript
|
||||
output_path: artifacts/session_bounds.json
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
notarius:
|
||||
# Optional structured extraction between trim and render.
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
working_directory: /usr/local/etc/notarius
|
||||
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||
outputs:
|
||||
item_registry:
|
||||
lane_id: item-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/item-registry
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
item_occurrences:
|
||||
lane_id: item-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/item-occurrences
|
||||
spells:
|
||||
lane_id: spells
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.spells
|
||||
schema_version: v1
|
||||
module_key: dnd/spells
|
||||
combat_turns:
|
||||
lane_id: combat-turns
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.combat_turns
|
||||
schema_version: v1
|
||||
module_key: dnd/combat-turns
|
||||
npc_occurrences:
|
||||
lane_id: npc-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-occurrences
|
||||
location_occurrences:
|
||||
lane_id: location-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/location-occurrences
|
||||
enemy_events:
|
||||
lane_id: enemy-events
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.enemy_events
|
||||
schema_version: v1
|
||||
module_key: dnd/enemy-events
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
# Configured artifact keys map to source IDs narratio.artifact.<key>.
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
players:
|
||||
source: narratio.input.players
|
||||
required: true
|
||||
party:
|
||||
source: narratio.input.party
|
||||
required: true
|
||||
glossary:
|
||||
source: narratio.input.glossary
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
|
||||
# Example dependent artifact:
|
||||
# - depends_on entries use artifact keys.
|
||||
# - narratio.artifact.<key> sources require matching depends_on membership.
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
campaign_name: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
# Optional notification settings.
|
||||
backend: ""
|
||||
recipient: ""
|
||||
timeout: 30s
|
||||
@@ -1,55 +1,6 @@
|
||||
workspace:
|
||||
root: ./tmp/narratio-workspace
|
||||
cleanup_after_archive: false
|
||||
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: "my-dnd-archive"
|
||||
root_prefix: "dnd"
|
||||
region: "us-east-1"
|
||||
# Optional credential env-var names (defaulted when omitted):
|
||||
# access_key_id_env: "OBJECT_STORAGE_KEY_ID"
|
||||
# secret_access_key_env: "OBJECT_STORAGE_KEY"
|
||||
|
||||
spool:
|
||||
root: "/var/spool/narratio"
|
||||
delete_audio_after_archive: false
|
||||
|
||||
archive:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
# Optional. When omitted entirely, Narratio defaults to seriatim binary + runtime defaults.
|
||||
seriatim: {}
|
||||
|
||||
# Optional runtime overrides. Model/provider can be owned by Audita runtime config.
|
||||
audita:
|
||||
config_path: "/usr/local/etc/audita/config.yml"
|
||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||
|
||||
# Optional Scriptorium integration for analyze artifacts.
|
||||
scriptorium:
|
||||
config_path: "/usr/local/etc/scriptorium/config.yml"
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
output_path: "artifacts/session_recap.md"
|
||||
inputs:
|
||||
transcript:
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
|
||||
128
examples/pipeline.production.yml
Normal file
128
examples/pipeline.production.yml
Normal file
@@ -0,0 +1,128 @@
|
||||
workspace:
|
||||
root: /var/lib/narratio/workspace
|
||||
cleanup_after_publish: true
|
||||
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: my-dnd-archive
|
||||
root_prefix: dnd
|
||||
region: us-east-1
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
root: /var/spool/narratio
|
||||
delete_audio_after_publish: true
|
||||
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
language: en
|
||||
timeout: 45m
|
||||
retries: 3
|
||||
retry_delay: 3s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
|
||||
audita:
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
players:
|
||||
source: narratio.input.players
|
||||
required: true
|
||||
party:
|
||||
source: narratio.input.party
|
||||
required: true
|
||||
glossary:
|
||||
source: narratio.input.glossary
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
timeout: 30s
|
||||
5
examples/session.local-audio.yml
Normal file
5
examples/session.local-audio.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
@@ -1,13 +0,0 @@
|
||||
session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
|
||||
# Narratio prepare lists this prefix and downloads .flac files.
|
||||
# audio_s3:
|
||||
# prefix: "audio/"
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
6
examples/session.s3-audio.yml
Normal file
6
examples/session.s3-audio.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
@@ -1,12 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
campaign: sample-campaign
|
||||
date: ""
|
||||
title: ""
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
|
||||
# audio_s3:
|
||||
# prefix: "audio/{{ session_id }}/"
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
|
||||
3
go.mod
3
go.mod
@@ -4,15 +4,16 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
|
||||
2
go.sum
2
go.sum
@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// NoopRunner is a deterministic no-op analyzer adapter.
|
||||
type NoopRunner struct{}
|
||||
|
||||
// Run returns the requested output path with placeholder metadata.
|
||||
func (n *NoopRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
return AnalyzeResult{ArtifactPath: req.OutputPath, Metadata: map[string]any{"placeholder": true}}, nil
|
||||
}
|
||||
|
||||
// FakeRunner captures analyze requests and returns deterministic responses.
|
||||
type FakeRunner struct {
|
||||
Requests []AnalyzeRequest
|
||||
Err error
|
||||
Result AnalyzeResult
|
||||
}
|
||||
|
||||
// Run records request and returns configured response.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return AnalyzeResult{}, f.Err
|
||||
}
|
||||
res := f.Result
|
||||
if res.ArtifactPath == "" {
|
||||
res.ArtifactPath = req.OutputPath
|
||||
}
|
||||
if res.Metadata == nil {
|
||||
res.Metadata = map[string]any{"fake": true}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
fake := &FakeRunner{}
|
||||
req := AnalyzeRequest{ArtifactType: "session-log", OutputPath: "artifacts/session-log.md"}
|
||||
|
||||
res, err := fake.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 || fake.Requests[0].ArtifactType != "session-log" {
|
||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
||||
}
|
||||
if res.ArtifactPath != req.OutputPath {
|
||||
t.Fatalf("artifact path = %q, want %q", res.ArtifactPath, req.OutputPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerError(t *testing.T) {
|
||||
fake := &FakeRunner{Err: errors.New("boom")}
|
||||
_, err := fake.Run(context.Background(), AnalyzeRequest{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Package analyzer declares the adapter contract for artifact analysis generation.
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// TODO: implement analyzer integration once the analyzer contract is finalized.
|
||||
|
||||
// Runner is the adapter boundary for analyzer invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error)
|
||||
}
|
||||
|
||||
// AnalyzeRequest describes one analyzer artifact generation request.
|
||||
type AnalyzeRequest struct {
|
||||
ArtifactType string
|
||||
ProcessedTranscriptPath string
|
||||
ContextReferences []string
|
||||
OutputPath string
|
||||
GeneratedConfigPath string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// AnalyzeResult describes analyzer output.
|
||||
type AnalyzeResult struct {
|
||||
ArtifactPath string
|
||||
Metadata map[string]any
|
||||
}
|
||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "audita.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "audita.stderr.log"),
|
||||
}
|
||||
|
||||
@@ -52,9 +52,9 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
ReportPath: filepath.Join(dir, "audita.report.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
@@ -571,7 +571,7 @@ func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunne
|
||||
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
merged := filepath.Join(dir, "merged.json")
|
||||
merged := filepath.Join(dir, "base.json")
|
||||
glossary := filepath.Join(dir, "glossary.yml")
|
||||
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, glossary, "terms: []\n")
|
||||
@@ -579,7 +579,7 @@ func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: merged,
|
||||
GlossaryPath: glossary,
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||
|
||||
22
internal/adapters/notarius/fake.go
Normal file
22
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package notarius
|
||||
|
||||
import "context"
|
||||
|
||||
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||
type FakeRunner struct {
|
||||
Requests []RunRequest
|
||||
Result RunResult
|
||||
Err error
|
||||
}
|
||||
|
||||
// Run records the request and returns the configured result or error.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return RunResult{}, f.Err
|
||||
}
|
||||
return f.Result, nil
|
||||
}
|
||||
108
internal/adapters/notarius/runner.go
Normal file
108
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ReceiptSchemaVersion = "notarius.run-result.v1"
|
||||
|
||||
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||
}
|
||||
|
||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||
type RunRequest struct {
|
||||
Binary string
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
InputPath string
|
||||
OutputRoot string
|
||||
WorkingDirectory string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Receipt is the transport-neutral successful run receipt.
|
||||
type Receipt struct {
|
||||
SchemaVersion string
|
||||
RunID string
|
||||
PipelineID string
|
||||
OutputDirectory string
|
||||
IndexFile string
|
||||
NormalizedOutputCount int
|
||||
RejectedOutputCount int
|
||||
WarningCount int
|
||||
ValidationStatus string
|
||||
DebugDirectory string
|
||||
}
|
||||
|
||||
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||
type LaneDescriptor struct {
|
||||
LaneID string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
ModuleKey string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||
type PipelineDescriptor struct {
|
||||
ArtifactKind string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// Index describes the validated bundle-management and artifact paths.
|
||||
type Index struct {
|
||||
Path string
|
||||
ManifestFile string
|
||||
ManifestPath string
|
||||
RejectedFile string
|
||||
RejectedPath string
|
||||
WarningsFile string
|
||||
WarningsPath string
|
||||
Lanes []LaneDescriptor
|
||||
ChunkMap *PipelineDescriptor
|
||||
EvidenceContext *PipelineDescriptor
|
||||
}
|
||||
|
||||
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||
type RejectionSummary struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ChunkID string
|
||||
ValidatorName string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// WarningSummary retains structured warning identity without free-form messages.
|
||||
type WarningSummary struct {
|
||||
Scope string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||
type RunResult struct {
|
||||
Receipt Receipt
|
||||
Index Index
|
||||
BundleRoot string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
Rejections []RejectionSummary
|
||||
Warnings []WarningSummary
|
||||
}
|
||||
524
internal/adapters/notarius/subprocess.go
Normal file
524
internal/adapters/notarius/subprocess.go
Normal file
@@ -0,0 +1,524 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
maxReceiptBytes = 1 << 20
|
||||
maxIndexBytes = 4 << 20
|
||||
maxSummaryBytes = 4 << 20
|
||||
canonicalIndexFile = "index.json"
|
||||
canonicalManifestFile = "manifest.json"
|
||||
canonicalRejectedFile = "rejected.json"
|
||||
canonicalWarningsFile = "warnings.json"
|
||||
)
|
||||
|
||||
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
||||
|
||||
// SubprocessRunner invokes Notarius through its public CLI.
|
||||
type SubprocessRunner struct {
|
||||
run subprocessRun
|
||||
}
|
||||
|
||||
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
||||
func NewSubprocessRunner() *SubprocessRunner {
|
||||
return &SubprocessRunner{run: subprocess.Run}
|
||||
}
|
||||
|
||||
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
||||
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if r == nil || r.run == nil {
|
||||
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
||||
}
|
||||
if err := validateRunRequest(req); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"run", req.PipelineID,
|
||||
"--config", req.ConfigPath,
|
||||
"--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot,
|
||||
"--json",
|
||||
}
|
||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDirectory,
|
||||
Timeout: req.Timeout,
|
||||
StdoutLogPath: req.ReceiptPath,
|
||||
StderrLogPath: req.LogPath,
|
||||
})
|
||||
baseResult := RunResult{
|
||||
ReceiptPath: req.ReceiptPath,
|
||||
LogPath: req.LogPath,
|
||||
ExitCode: processResult.ExitCode,
|
||||
Duration: processResult.Duration,
|
||||
}
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
||||
}
|
||||
|
||||
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
||||
}
|
||||
index, err := loadIndex(bundleRoot, indexPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
rejections, err := loadRejections(index.RejectedPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
warnings, err := loadWarnings(index.WarningsPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
|
||||
baseResult.Receipt = receipt
|
||||
baseResult.Index = index
|
||||
baseResult.BundleRoot = bundleRoot
|
||||
baseResult.Rejections = rejections
|
||||
baseResult.Warnings = warnings
|
||||
return baseResult, nil
|
||||
}
|
||||
|
||||
func validateRunRequest(req RunRequest) error {
|
||||
if strings.TrimSpace(req.Binary) == "" {
|
||||
return fmt.Errorf("notarius binary is required")
|
||||
}
|
||||
if strings.TrimSpace(req.PipelineID) == "" {
|
||||
return fmt.Errorf("notarius pipeline id is required")
|
||||
}
|
||||
if req.Timeout <= 0 {
|
||||
return fmt.Errorf("notarius timeout must be positive")
|
||||
}
|
||||
for label, path := range map[string]string{
|
||||
"config": req.ConfigPath,
|
||||
"input": req.InputPath,
|
||||
"output root": req.OutputRoot,
|
||||
"working directory": req.WorkingDirectory,
|
||||
"receipt": req.ReceiptPath,
|
||||
"log": req.LogPath,
|
||||
} {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("notarius %s path is required", label)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("notarius %s path must be absolute", label)
|
||||
}
|
||||
}
|
||||
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
||||
return fmt.Errorf("notarius receipt and log paths must be different")
|
||||
}
|
||||
if err := requireRegularFile(req.ConfigPath); err != nil {
|
||||
return fmt.Errorf("validate notarius config path: %w", err)
|
||||
}
|
||||
if err := requireRegularFile(req.InputPath); err != nil {
|
||||
return fmt.Errorf("validate notarius input path: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.OutputRoot); err != nil {
|
||||
return fmt.Errorf("validate notarius output root: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
||||
return fmt.Errorf("validate notarius working directory: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
||||
return fmt.Errorf("validate notarius receipt path: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.LogPath); err != nil {
|
||||
return fmt.Errorf("validate notarius log path: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type receiptDocument struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file"`
|
||||
NormalizedOutputCount *int `json:"normalized_output_count"`
|
||||
RejectedOutputCount *int `json:"rejected_output_count"`
|
||||
WarningCount *int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
DebugDirectory string `json:"debug_directory"`
|
||||
}
|
||||
|
||||
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
||||
var document receiptDocument
|
||||
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
||||
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
||||
}
|
||||
if document.SchemaVersion != ReceiptSchemaVersion {
|
||||
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
||||
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
||||
document.NormalizedOutputCount == nil ||
|
||||
document.RejectedOutputCount == nil || document.WarningCount == nil {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
||||
}
|
||||
if document.IndexFile != canonicalIndexFile {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
||||
}
|
||||
if document.PipelineID != pipelineID {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
||||
}
|
||||
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
||||
}
|
||||
if !filepath.IsAbs(document.OutputDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
||||
}
|
||||
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
||||
}
|
||||
return Receipt{
|
||||
SchemaVersion: document.SchemaVersion,
|
||||
RunID: document.RunID,
|
||||
PipelineID: document.PipelineID,
|
||||
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
||||
IndexFile: document.IndexFile,
|
||||
NormalizedOutputCount: *document.NormalizedOutputCount,
|
||||
RejectedOutputCount: *document.RejectedOutputCount,
|
||||
WarningCount: *document.WarningCount,
|
||||
ValidationStatus: document.ValidationStatus,
|
||||
DebugDirectory: document.DebugDirectory,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type indexDocument struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles *[]laneDocument `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
ChunkMap *pipelineDocument `json:"chunk_map"`
|
||||
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
||||
}
|
||||
|
||||
type laneDocument struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
type pipelineDocument struct {
|
||||
ArtifactKind string `json:"artifact_kind"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
||||
var document indexDocument
|
||||
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
||||
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
got string
|
||||
want string
|
||||
}{
|
||||
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
||||
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
||||
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
||||
} {
|
||||
if field.got != field.want {
|
||||
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
||||
}
|
||||
}
|
||||
if document.OutputFiles == nil {
|
||||
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
||||
}
|
||||
|
||||
index := Index{
|
||||
Path: indexPath,
|
||||
ManifestFile: document.ManifestFile,
|
||||
RejectedFile: document.RejectedFile,
|
||||
WarningsFile: document.WarningsFile,
|
||||
}
|
||||
var err error
|
||||
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
||||
}
|
||||
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
||||
}
|
||||
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
||||
}
|
||||
|
||||
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
||||
for _, lane := range *document.OutputFiles {
|
||||
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
||||
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
||||
}
|
||||
if _, exists := seenLanes[lane.LaneID]; exists {
|
||||
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
||||
}
|
||||
seenLanes[lane.LaneID] = struct{}{}
|
||||
path, err := resolveRegularFile(bundleRoot, lane.File)
|
||||
if err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
||||
}
|
||||
index.Lanes = append(index.Lanes, LaneDescriptor{
|
||||
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
||||
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
||||
SchemaVersion: lane.SchemaVersion,
|
||||
})
|
||||
}
|
||||
if document.ChunkMap != nil {
|
||||
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
if document.EvidenceContext != nil {
|
||||
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
||||
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
||||
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
||||
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
||||
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
||||
}
|
||||
path, err := resolveRegularFile(bundleRoot, document.File)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
||||
}
|
||||
return &PipelineDescriptor{
|
||||
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
||||
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
||||
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type rejectionDocument struct {
|
||||
Rejected *[]struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"rejected"`
|
||||
}
|
||||
|
||||
func loadRejections(path string) ([]RejectionSummary, error) {
|
||||
var document rejectionDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
||||
}
|
||||
if document.Rejected == nil {
|
||||
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
||||
}
|
||||
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
||||
for _, item := range *document.Rejected {
|
||||
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
||||
}
|
||||
summaries = append(summaries, RejectionSummary{
|
||||
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
||||
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
||||
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
||||
})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
type warningDocument struct {
|
||||
Warnings *[]struct {
|
||||
Scope string `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"warnings"`
|
||||
}
|
||||
|
||||
func loadWarnings(path string) ([]WarningSummary, error) {
|
||||
var document warningDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
||||
}
|
||||
if document.Warnings == nil {
|
||||
return nil, fmt.Errorf("notarius warning document is missing warnings array")
|
||||
}
|
||||
summaries := make([]WarningSummary, 0, len(*document.Warnings))
|
||||
for _, item := range *document.Warnings {
|
||||
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
|
||||
}
|
||||
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||
inspected, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inspected.Mode()&os.ModeSymlink != 0 || !inspected.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
opened, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) {
|
||||
return fmt.Errorf("file %q changed before it could be read", path)
|
||||
}
|
||||
reader := io.LimitReader(file, limit+1)
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return fmt.Errorf("file %q exceeds %d-byte limit", path, limit)
|
||||
}
|
||||
if err := json.Unmarshal(data, destination); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
||||
root := filepath.Clean(outputRoot)
|
||||
bundle := filepath.Clean(bundleRoot)
|
||||
relative, err := filepath.Rel(root, bundle)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
||||
}
|
||||
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
||||
}
|
||||
if err := requireDirectoryTree(root, relative); err != nil {
|
||||
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
||||
}
|
||||
return bundle, nil
|
||||
}
|
||||
|
||||
func resolveRegularFile(root, logicalPath string) (string, error) {
|
||||
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
relative, err := filepath.Rel(root, resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := requireRegularFileTree(root, relative); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func requireDirectoryTree(root, relative string) error {
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFileTree(root, relative string) error {
|
||||
components := strings.Split(relative, string(filepath.Separator))
|
||||
if len(components) == 0 {
|
||||
return fmt.Errorf("regular file path is required")
|
||||
}
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range components[:len(components)-1] {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
||||
}
|
||||
|
||||
func requireDirectory(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFile(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLogDestination(path string) error {
|
||||
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
572
internal/adapters/notarius/subprocess_test.go
Normal file
572
internal/adapters/notarius/subprocess_test.go
Normal file
@@ -0,0 +1,572 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
var captured sharedsubprocess.RunRequest
|
||||
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeValidBundleAndReceipt(t, req, true)
|
||||
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
|
||||
}}
|
||||
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
wantArgs := []string{
|
||||
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot, "--json",
|
||||
}
|
||||
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||
}
|
||||
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
|
||||
t.Fatalf("subprocess request = %#v", captured)
|
||||
}
|
||||
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
|
||||
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
|
||||
}
|
||||
if captured.EnvOverrides != nil {
|
||||
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
|
||||
}
|
||||
for _, arg := range captured.Args {
|
||||
if arg == "--session-id" {
|
||||
t.Fatal("subprocess args unexpectedly contain --session-id")
|
||||
}
|
||||
}
|
||||
|
||||
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
|
||||
t.Fatalf("receipt = %#v", result.Receipt)
|
||||
}
|
||||
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
|
||||
t.Fatalf("lanes = %#v", result.Index.Lanes)
|
||||
}
|
||||
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
|
||||
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
|
||||
}
|
||||
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
|
||||
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
|
||||
}
|
||||
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
|
||||
t.Fatalf("rejections = %#v", result.Rejections)
|
||||
}
|
||||
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
|
||||
t.Fatalf("warnings = %#v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInheritsEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
receiptFixture := req.ReceiptPath + ".fixture"
|
||||
data, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||
}
|
||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||
t.Fatalf("Remove(receipt) error = %v", err)
|
||||
}
|
||||
|
||||
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
|
||||
if err := os.Mkdir(captureDir, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(capture) error = %v", err)
|
||||
}
|
||||
script := writeShellScript(t, `#!/bin/sh
|
||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||
printf 'diagnostic stream\n' >&2
|
||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||
`)
|
||||
req.Binary = script
|
||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||
|
||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "inherited-value")
|
||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||
t.Fatal("receipt contains stderr output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scriptBody string
|
||||
timeout time.Duration
|
||||
cancel bool
|
||||
want string
|
||||
}{
|
||||
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
|
||||
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
|
||||
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||
req.Timeout = test.timeout
|
||||
ctx := context.Background()
|
||||
if test.cancel {
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
ctx = cancelCtx
|
||||
time.AfterFunc(20*time.Millisecond, cancel)
|
||||
}
|
||||
_, err := NewSubprocessRunner().Run(ctx, req)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode notarius receipt") {
|
||||
t.Fatalf("Run() parsed stdout after process failure: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt) error = %v", err)
|
||||
}
|
||||
wantErr := errors.New("process failed")
|
||||
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
|
||||
}}
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run() error = %v, want wrapped process error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode") {
|
||||
t.Fatalf("Run() parsed receipt after failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReceiptValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
valid := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
|
||||
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
|
||||
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
|
||||
"validation_status": "approved", "future_field": true,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(map[string]any)
|
||||
raw []byte
|
||||
wantOK bool
|
||||
wantError string
|
||||
}{
|
||||
{name: "unknown fields tolerated", wantOK: true},
|
||||
{name: "malformed", raw: []byte("{")},
|
||||
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
|
||||
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
|
||||
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
|
||||
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
|
||||
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
|
||||
{
|
||||
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
|
||||
wantError: `index_file "nested/index.json"`,
|
||||
},
|
||||
{
|
||||
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
|
||||
wantError: `index_file "./index.json"`,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||
values := cloneMap(valid)
|
||||
if test.mutate != nil {
|
||||
test.mutate(values)
|
||||
}
|
||||
if test.raw != nil {
|
||||
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, path, values)
|
||||
}
|
||||
_, err := loadReceipt(path, "pipeline-1")
|
||||
if test.wantOK && err != nil {
|
||||
t.Fatalf("loadReceipt() error = %v", err)
|
||||
}
|
||||
if !test.wantOK && err == nil {
|
||||
t.Fatal("loadReceipt() error = nil, want validation failure")
|
||||
}
|
||||
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadReceipt(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outputRoot := filepath.Join(root, "output")
|
||||
if err := os.Mkdir(outputRoot, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(output root) error = %v", err)
|
||||
}
|
||||
validBundle := filepath.Join(outputRoot, "run-1")
|
||||
if err := os.Mkdir(validBundle, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(bundle) error = %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
|
||||
t.Fatalf("validateBundleRoot(valid) error = %v", err)
|
||||
}
|
||||
|
||||
outside := filepath.Join(root, "output-other")
|
||||
if err := os.Mkdir(outside, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(outside) error = %v", err)
|
||||
}
|
||||
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
|
||||
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
|
||||
}
|
||||
})
|
||||
}
|
||||
symlink := filepath.Join(outputRoot, "linked")
|
||||
if err := os.Symlink(outside, symlink); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
|
||||
t.Fatal("validateBundleRoot(symlink) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
indexValue any
|
||||
prepare func(*testing.T, string)
|
||||
wantError string
|
||||
}{
|
||||
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
|
||||
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "renamed manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "metadata.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "metadata.json"`},
|
||||
{name: "cleanable manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "./manifest.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "./manifest.json"`},
|
||||
{name: "renamed rejections", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["rejected_file"] = "rejections.json"
|
||||
return value
|
||||
}(), wantError: `rejected_file "rejections.json"`},
|
||||
{name: "renamed warnings", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["warnings_file"] = "diagnostics/warnings.json"
|
||||
return value
|
||||
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
|
||||
{name: "duplicate lane", indexValue: validIndexValue([]any{
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
})},
|
||||
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
|
||||
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
|
||||
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
|
||||
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
|
||||
t.Fatalf("Remove(manifest) error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "incomplete pipeline descriptor", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
|
||||
return value
|
||||
}()},
|
||||
{name: "pipeline descriptor escape", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
|
||||
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
|
||||
}
|
||||
return value
|
||||
}()},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
bundle := createBundleSkeleton(t)
|
||||
indexPath := filepath.Join(bundle, "index.json")
|
||||
if raw, ok := test.indexValue.(json.RawMessage); ok {
|
||||
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(index) error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, indexPath, test.indexValue)
|
||||
}
|
||||
if test.prepare != nil {
|
||||
test.prepare(t, bundle)
|
||||
}
|
||||
if _, err := loadIndex(bundle, indexPath); err == nil {
|
||||
t.Fatal("loadIndex() error = nil, want failure")
|
||||
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
bundle := createBundleSkeleton(t)
|
||||
oversizedIndex := filepath.Join(bundle, "index.json")
|
||||
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized index) error = %v", err)
|
||||
}
|
||||
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadIndex(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rejectedPath := filepath.Join(root, "rejected.json")
|
||||
warningsPath := filepath.Join(root, "warnings.json")
|
||||
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
|
||||
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
|
||||
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
rejections, err := loadRejections(rejectedPath)
|
||||
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
|
||||
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
|
||||
}
|
||||
warnings, err := loadWarnings(warningsPath)
|
||||
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
|
||||
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
|
||||
}
|
||||
|
||||
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
|
||||
t.Run("malformed "+name, func(t *testing.T) {
|
||||
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
var err error
|
||||
if name == "rejections" {
|
||||
_, err = loadRejections(path)
|
||||
} else {
|
||||
_, err = loadWarnings(path)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("summary decoder error = nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadWarnings(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadRejections(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||
req := RunRequest{PipelineID: "pipeline"}
|
||||
want := RunResult{BundleRoot: "/bundle"}
|
||||
fake := &FakeRunner{Result: want}
|
||||
got, err := fake.Run(context.Background(), req)
|
||||
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
|
||||
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
|
||||
}
|
||||
|
||||
wantErr := errors.New("configured failure")
|
||||
fake.Err = wantErr
|
||||
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run(configured error) = %v", err)
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
before := len(fake.Requests)
|
||||
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
|
||||
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
|
||||
}
|
||||
}
|
||||
|
||||
func validRunRequest(t *testing.T) RunRequest {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
inputPath := filepath.Join(root, "input.json")
|
||||
outputRoot := filepath.Join(root, "outputs")
|
||||
workingDirectory := filepath.Join(root, "work")
|
||||
diagnostics := filepath.Join(root, "diagnostics")
|
||||
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
|
||||
if err := os.Mkdir(directory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(%q) error = %v", directory, err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
return RunRequest{
|
||||
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
|
||||
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
|
||||
Timeout: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for path, data := range map[string]string{
|
||||
"manifest.json": `{}`,
|
||||
"lanes/npc.json": `{}`,
|
||||
"chunk-map.json": `{}`,
|
||||
"evidence-context.json": `{}`,
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
|
||||
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
|
||||
if includeUnknown {
|
||||
rejection["future"] = true
|
||||
warning["future"] = true
|
||||
}
|
||||
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
|
||||
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
|
||||
index := validIndexValue([]any{map[string]any{
|
||||
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
|
||||
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
|
||||
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
|
||||
}})
|
||||
index["chunk_map"] = map[string]any{
|
||||
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["future"] = true
|
||||
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
|
||||
receipt := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
|
||||
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
|
||||
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
|
||||
}
|
||||
if includeUnknown {
|
||||
receipt["future"] = true
|
||||
}
|
||||
writeJSONFile(t, req.ReceiptPath, receipt)
|
||||
}
|
||||
|
||||
func createBundleSkeleton(t *testing.T) string {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", name, err)
|
||||
}
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
func validIndexValue(lanes []any) map[string]any {
|
||||
return map[string]any{
|
||||
"manifest_file": "manifest.json", "output_files": lanes,
|
||||
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSONFile(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeShellScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "notarius-helper")
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(script) error = %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func assertTextFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]any) map[string]any {
|
||||
result := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -31,7 +31,7 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
||||
ConfigPath: "/etc/scriptorium/config.yml",
|
||||
PromptID: "dnd.session_recap",
|
||||
ProfileID: "local-quality",
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json"), "other": filepath.Join(dir, "other.md")},
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json"), "other": filepath.Join(dir, "other.md")},
|
||||
Vars: map[string]string{"session_id": "2026-05-03", "campaign_name": "Icewind Dale"},
|
||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
||||
@@ -180,7 +180,7 @@ func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||
req := RenderArtifactRequest{
|
||||
Binary: wrapper,
|
||||
PromptID: "dnd.session_recap",
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json")},
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json")},
|
||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.render.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.render.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.render.stderr.log"),
|
||||
@@ -285,7 +285,7 @@ type scriptoriumHelperRecord struct {
|
||||
func runReqForTest(t *testing.T, binary string) RunArtifactRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "processed.json")
|
||||
transcriptPath := filepath.Join(dir, "polished.json")
|
||||
writeScriptoriumFile(t, transcriptPath, `{"segments":[]}`)
|
||||
return RunArtifactRequest{
|
||||
Binary: binary,
|
||||
|
||||
@@ -68,6 +68,26 @@ func (n *NoopRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Render returns the requested output path with placeholder metadata.
|
||||
func (n *NoopRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RenderResult{}, err
|
||||
}
|
||||
if err := materializeRenderPlaceholders(req); err != nil {
|
||||
return RenderResult{}, err
|
||||
}
|
||||
return RenderResult{
|
||||
OutputRenderedPath: req.OutputRenderedPath,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||
InvokedBinary: "noop",
|
||||
Format: req.Format,
|
||||
Title: req.Title,
|
||||
Metadata: map[string]any{"placeholder": true},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FakeRunner captures merge requests and returns deterministic responses.
|
||||
type FakeRunner struct {
|
||||
Requests []MergeRequest
|
||||
@@ -79,6 +99,9 @@ type FakeRunner struct {
|
||||
TrimRequests []TrimRequest
|
||||
TrimErr error
|
||||
TrimResult TrimResult
|
||||
RenderRequests []RenderRequest
|
||||
RenderErr error
|
||||
RenderResult RenderResult
|
||||
}
|
||||
|
||||
// Run records request and returns configured response.
|
||||
@@ -195,6 +218,46 @@ func (f *FakeRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Render records request and returns configured response.
|
||||
func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RenderResult{}, err
|
||||
}
|
||||
f.RenderRequests = append(f.RenderRequests, req)
|
||||
if f.RenderErr != nil {
|
||||
return RenderResult{}, f.RenderErr
|
||||
}
|
||||
if err := materializeRenderPlaceholders(req); err != nil {
|
||||
return RenderResult{}, err
|
||||
}
|
||||
res := f.RenderResult
|
||||
if res.OutputRenderedPath == "" {
|
||||
res.OutputRenderedPath = req.OutputRenderedPath
|
||||
}
|
||||
if res.StdoutLogPath == "" {
|
||||
res.StdoutLogPath = req.StdoutLogPath
|
||||
}
|
||||
if res.StderrLogPath == "" {
|
||||
res.StderrLogPath = req.StderrLogPath
|
||||
}
|
||||
if res.GeneratedConfigPath == "" {
|
||||
res.GeneratedConfigPath = req.GeneratedConfigPath
|
||||
}
|
||||
if res.InvokedBinary == "" {
|
||||
res.InvokedBinary = "fake"
|
||||
}
|
||||
if res.Format == "" {
|
||||
res.Format = req.Format
|
||||
}
|
||||
if res.Title == "" {
|
||||
res.Title = req.Title
|
||||
}
|
||||
if res.Metadata == nil {
|
||||
res.Metadata = map[string]any{"fake": true}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func materializePlaceholders(req MergeRequest) error {
|
||||
if req.OutputMergedTranscriptPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
||||
@@ -301,3 +364,39 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
if req.OutputRenderedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||
}
|
||||
}
|
||||
if req.GeneratedConfigPath != "" {
|
||||
payload := map[string]any{
|
||||
"schema": "seriatim.generated.v1",
|
||||
"placeholder": true,
|
||||
"command": "render",
|
||||
"input_path": req.InputTranscriptPath,
|
||||
"output_path": req.OutputRenderedPath,
|
||||
"format": req.Format,
|
||||
"title": req.Title,
|
||||
"include_timestamps": req.IncludeTimestamps,
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.yml"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "base.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.stderr.log"),
|
||||
}
|
||||
@@ -57,8 +57,8 @@ func TestFakeRunnerTrimCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := TrimRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.trim.yml"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "transcripts", "trimmed.json"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
|
||||
KeepSelector: "1-10",
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.trim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.trim.stderr.log"),
|
||||
@@ -105,8 +105,8 @@ func TestFakeRunnerNormalizeCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := NormalizeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.normalize.yml"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "transcripts", "normalized.json"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "transcripts", "final.json"),
|
||||
OutputSchema: "seriatim-intermediate",
|
||||
ReportPath: filepath.Join(dir, "artifacts", "seriatim.normalize.report.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.normalize.stdout.log"),
|
||||
@@ -148,3 +148,58 @@ func TestFakeRunnerNormalizeError(t *testing.T) {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerRenderCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
fake := &FakeRunner{}
|
||||
dir := t.TempDir()
|
||||
req := RenderRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.render.yml"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
|
||||
OutputRenderedPath: filepath.Join(dir, "transcripts", "final.trimmed.md"),
|
||||
Format: "markdown",
|
||||
Title: "Session render",
|
||||
IncludeTimestamps: true,
|
||||
IncludeSegmentIDs: false,
|
||||
IncludeMetadata: true,
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.render.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.render.stderr.log"),
|
||||
}
|
||||
|
||||
res, err := fake.Render(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
if len(fake.RenderRequests) != 1 || fake.RenderRequests[0].GeneratedConfigPath == "" {
|
||||
t.Fatalf("render requests = %#v, want captured request", fake.RenderRequests)
|
||||
}
|
||||
if res.OutputRenderedPath != req.OutputRenderedPath {
|
||||
t.Fatalf("rendered path = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
|
||||
}
|
||||
if res.Format != req.Format {
|
||||
t.Fatalf("format = %q, want %q", res.Format, req.Format)
|
||||
}
|
||||
if res.Title != req.Title {
|
||||
t.Fatalf("title = %q, want %q", res.Title, req.Title)
|
||||
}
|
||||
|
||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read generated config: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(cfgData), "command: render") {
|
||||
t.Fatalf("generated config = %q, want render command marker", string(cfgData))
|
||||
}
|
||||
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath, req.OutputRenderedPath} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected file %q to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerRenderError(t *testing.T) {
|
||||
fake := &FakeRunner{RenderErr: errors.New("boom")}
|
||||
_, err := fake.Render(context.Background(), RenderRequest{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Package seriatim declares the adapter contract for transcript merge/normalize/trim execution.
|
||||
// Package seriatim declares the adapter contract for transcript merge/normalize/trim/render execution.
|
||||
package seriatim
|
||||
|
||||
import (
|
||||
@@ -6,11 +6,12 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Runner is the adapter boundary for seriatim merge/normalize/trim invocations.
|
||||
// Runner is the adapter boundary for seriatim merge/normalize/trim/render invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req MergeRequest) (MergeResult, error)
|
||||
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
||||
Trim(ctx context.Context, req TrimRequest) (TrimResult, error)
|
||||
Render(ctx context.Context, req RenderRequest) (RenderResult, error)
|
||||
}
|
||||
|
||||
// MergeRequest describes a seriatim merge invocation.
|
||||
@@ -90,3 +91,33 @@ type TrimResult struct {
|
||||
KeepSelector string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
// RenderRequest describes a seriatim render invocation.
|
||||
type RenderRequest struct {
|
||||
Binary string
|
||||
InputTranscriptPath string
|
||||
OutputRenderedPath string
|
||||
Format string
|
||||
Title string
|
||||
IncludeTimestamps bool
|
||||
IncludeSegmentIDs bool
|
||||
IncludeMetadata bool
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
GeneratedConfigPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// RenderResult describes a render output.
|
||||
type RenderResult struct {
|
||||
OutputRenderedPath string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
GeneratedConfigPath string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
InvokedBinary string
|
||||
Format string
|
||||
Title string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
@@ -384,6 +385,96 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Render executes Seriatim render with deterministic flags and validates non-empty text output.
|
||||
func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||
if r == nil {
|
||||
return RenderResult{}, fmt.Errorf("seriatim subprocess runner is nil")
|
||||
}
|
||||
if strings.TrimSpace(req.InputTranscriptPath) == "" {
|
||||
return RenderResult{}, fmt.Errorf("seriatim render input path is required")
|
||||
}
|
||||
if strings.TrimSpace(req.OutputRenderedPath) == "" {
|
||||
return RenderResult{}, fmt.Errorf("seriatim render output path is required")
|
||||
}
|
||||
format := strings.TrimSpace(req.Format)
|
||||
if format == "" {
|
||||
format = "markdown"
|
||||
}
|
||||
if format != "markdown" {
|
||||
return RenderResult{}, fmt.Errorf("seriatim render format %q is unsupported", req.Format)
|
||||
}
|
||||
|
||||
binary := r.binary
|
||||
if strings.TrimSpace(req.Binary) != "" {
|
||||
binary = strings.TrimSpace(req.Binary)
|
||||
}
|
||||
|
||||
timeout := r.timeout
|
||||
if req.Timeout < 0 {
|
||||
return RenderResult{}, fmt.Errorf("seriatim render timeout must be >= 0")
|
||||
}
|
||||
if req.Timeout > 0 {
|
||||
timeout = req.Timeout
|
||||
}
|
||||
|
||||
args := buildRenderArgs(req, format)
|
||||
if req.GeneratedConfigPath != "" {
|
||||
if err := writeRenderInvocationConfig(req, args, binary, timeout, format); err != nil {
|
||||
return RenderResult{}, fmt.Errorf("write seriatim render invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
return RenderResult{
|
||||
OutputRenderedPath: req.OutputRenderedPath,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||
ExitCode: runRes.ExitCode,
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: binary,
|
||||
Format: format,
|
||||
Title: req.Title,
|
||||
}, fmt.Errorf("run seriatim render (binary=%q): %w", binary, err)
|
||||
}
|
||||
|
||||
if err := validateNonEmptyTextFile(req.OutputRenderedPath); err != nil {
|
||||
return RenderResult{
|
||||
OutputRenderedPath: req.OutputRenderedPath,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||
ExitCode: runRes.ExitCode,
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: binary,
|
||||
Format: format,
|
||||
Title: req.Title,
|
||||
}, fmt.Errorf("validate seriatim rendered output %q: %w", req.OutputRenderedPath, err)
|
||||
}
|
||||
|
||||
return RenderResult{
|
||||
OutputRenderedPath: req.OutputRenderedPath,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||
ExitCode: runRes.ExitCode,
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: binary,
|
||||
Format: format,
|
||||
Title: req.Title,
|
||||
Metadata: map[string]any{
|
||||
"adapter": "seriatim_subprocess",
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) buildMergeArgs(req MergeRequest) []string {
|
||||
args := []string{"merge"}
|
||||
|
||||
@@ -480,6 +571,22 @@ func buildNormalizeArgs(req NormalizeRequest, outputSchema string) []string {
|
||||
return args
|
||||
}
|
||||
|
||||
func buildRenderArgs(req RenderRequest, format string) []string {
|
||||
args := []string{
|
||||
"render",
|
||||
"--input-file", req.InputTranscriptPath,
|
||||
"--output-file", req.OutputRenderedPath,
|
||||
"--format", format,
|
||||
"--include-timestamps=" + strconv.FormatBool(req.IncludeTimestamps),
|
||||
"--include-segment-ids=" + strconv.FormatBool(req.IncludeSegmentIDs),
|
||||
"--include-metadata=" + strconv.FormatBool(req.IncludeMetadata),
|
||||
}
|
||||
if strings.TrimSpace(req.Title) != "" {
|
||||
args = append(args, "--title", req.Title)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, timeout time.Duration) error {
|
||||
payload := map[string]any{
|
||||
"schema": "seriatim.generated.v1",
|
||||
@@ -509,6 +616,24 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
}
|
||||
|
||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||
payload := map[string]any{
|
||||
"schema": "seriatim.generated.v1",
|
||||
"command": "render",
|
||||
"binary": binary,
|
||||
"args": args,
|
||||
"timeout": timeout.String(),
|
||||
"input_path": req.InputTranscriptPath,
|
||||
"output_path": req.OutputRenderedPath,
|
||||
"format": format,
|
||||
"title": req.Title,
|
||||
"include_timestamps": req.IncludeTimestamps,
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -541,3 +666,20 @@ func validateJSONFileWithSegments(path string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateNonEmptyTextFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("file is empty")
|
||||
}
|
||||
if !utf8.Valid(data) {
|
||||
return fmt.Errorf("file is not valid utf-8 text")
|
||||
}
|
||||
if strings.TrimSpace(string(data)) == "" {
|
||||
return fmt.Errorf("file has no non-whitespace content")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||
InputTranscriptPaths: []string{filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json")},
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
ReportPath: filepath.Join(dir, "seriatim.report.json"),
|
||||
SpeakersPath: filepath.Join(dir, "speakers.yml"),
|
||||
AutocorrectPath: filepath.Join(dir, "autocorrect.yml"),
|
||||
@@ -569,6 +569,156 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
wrapper := writeHelperWrapper(t)
|
||||
runner := mustRunner(t, wrapper, false)
|
||||
req := renderReqForTest(t)
|
||||
|
||||
res, err := runner.Render(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
if res.OutputRenderedPath != req.OutputRenderedPath {
|
||||
t.Fatalf("OutputRenderedPath = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
|
||||
}
|
||||
if res.Format != req.Format {
|
||||
t.Fatalf("Format = %q, want %q", res.Format, req.Format)
|
||||
}
|
||||
if res.Title != req.Title {
|
||||
t.Fatalf("Title = %q, want %q", res.Title, req.Title)
|
||||
}
|
||||
if res.InvokedBinary != wrapper {
|
||||
t.Fatalf("InvokedBinary = %q, want %q", res.InvokedBinary, wrapper)
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||
}
|
||||
if res.Duration <= 0 {
|
||||
t.Fatalf("Duration = %s, want >0", res.Duration)
|
||||
}
|
||||
if res.Metadata == nil || res.Metadata["adapter"] != "seriatim_subprocess" {
|
||||
t.Fatalf("Metadata = %#v, want adapter marker", res.Metadata)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(req.OutputRenderedPath); err != nil {
|
||||
t.Fatalf("rendered output missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(req.StdoutLogPath); err != nil {
|
||||
t.Fatalf("stdout log missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(req.StderrLogPath); err != nil {
|
||||
t.Fatalf("stderr log missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(req.GeneratedConfigPath); err != nil {
|
||||
t.Fatalf("generated config missing: %v", err)
|
||||
}
|
||||
|
||||
rec := readHelperRecord(t, recordPath)
|
||||
wantArgs := []string{
|
||||
"render",
|
||||
"--input-file", req.InputTranscriptPath,
|
||||
"--output-file", req.OutputRenderedPath,
|
||||
"--format", req.Format,
|
||||
"--include-timestamps=true",
|
||||
"--include-segment-ids=true",
|
||||
"--include-metadata=false",
|
||||
"--title", req.Title,
|
||||
}
|
||||
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
|
||||
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||
req := renderReqForTest(t)
|
||||
req.Title = ""
|
||||
if _, err := runner.Render(context.Background(), req); err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
|
||||
rec := readHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--title" {
|
||||
t.Fatalf("args = %#v, did not expect --title", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||
req := renderReqForTest(t)
|
||||
_, err := runner.Render(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Render() error = nil, want non-nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "run seriatim render") {
|
||||
t.Fatalf("error = %q, want subprocess context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||
req := renderReqForTest(t)
|
||||
_, err := runner.Render(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Render() error = nil, want non-nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "validate seriatim rendered output") {
|
||||
t.Fatalf("error = %q, want output validation context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||
req := renderReqForTest(t)
|
||||
_, err := runner.Render(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Render() error = nil, want non-nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "file is empty") {
|
||||
t.Fatalf("error = %q, want empty-file validation", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "10m", "seriatim-intermediate", nil, true, EnvConfig{})
|
||||
if err == nil {
|
||||
@@ -702,6 +852,14 @@ func TestSeriatimSubprocessHelper(t *testing.T) {
|
||||
case "normalize_report_missing":
|
||||
writeSeriatimHelperFile(outputPath, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||
os.Exit(0)
|
||||
case "render_success":
|
||||
writeSeriatimHelperFile(outputPath, "# Rendered transcript\n\nHello.\n")
|
||||
_, _ = os.Stdout.WriteString("seriatim helper render stdout\n")
|
||||
_, _ = os.Stderr.WriteString("seriatim helper render stderr\n")
|
||||
os.Exit(0)
|
||||
case "render_empty_output":
|
||||
writeSeriatimHelperFile(outputPath, "")
|
||||
os.Exit(0)
|
||||
default:
|
||||
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
|
||||
os.Exit(2)
|
||||
@@ -732,7 +890,7 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||
InputTranscriptPaths: []string{in1, in2},
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "seriatim.stderr.log"),
|
||||
}
|
||||
@@ -745,11 +903,11 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
||||
func trimReqForTest(t *testing.T) TrimRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "processed.json")
|
||||
input := filepath.Join(dir, "polished.json")
|
||||
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||
return TrimRequest{
|
||||
InputTranscriptPath: input,
|
||||
OutputTrimmedPath: filepath.Join(dir, "trimmed.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "final.trimmed.json"),
|
||||
KeepSelector: "5-12",
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.trim.generated.yml"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.trim.stdout.log"),
|
||||
@@ -760,12 +918,12 @@ func trimReqForTest(t *testing.T) TrimRequest {
|
||||
func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "processed.json")
|
||||
input := filepath.Join(dir, "polished.json")
|
||||
writeSeriatimFile(t, input, `{"schema":"audita.processed.v1","segments":[]}`)
|
||||
|
||||
req := NormalizeRequest{
|
||||
InputTranscriptPath: input,
|
||||
OutputNormalizedPath: filepath.Join(dir, "normalized.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "final.json"),
|
||||
OutputSchema: "seriatim-intermediate",
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.normalize.generated.yml"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.normalize.stdout.log"),
|
||||
@@ -777,6 +935,25 @@ func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
||||
return req
|
||||
}
|
||||
|
||||
func renderReqForTest(t *testing.T) RenderRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "final.trimmed.json")
|
||||
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||
return RenderRequest{
|
||||
InputTranscriptPath: input,
|
||||
OutputRenderedPath: filepath.Join(dir, "final.trimmed.md"),
|
||||
Format: "markdown",
|
||||
Title: "Session 42",
|
||||
IncludeTimestamps: true,
|
||||
IncludeSegmentIDs: true,
|
||||
IncludeMetadata: false,
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.render.generated.yml"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.render.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "seriatim.render.stderr.log"),
|
||||
}
|
||||
}
|
||||
|
||||
func mustRunner(t *testing.T, binary string, report bool) *SubprocessRunner {
|
||||
t.Helper()
|
||||
coalesce := 3.0
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// Package storage declares archive/storage backend adapter boundaries.
|
||||
package storage
|
||||
|
||||
import "context"
|
||||
|
||||
// TODO: implement remote storage/archive backends (S3/SFTP/etc.).
|
||||
|
||||
// Backend is the adapter boundary for archive/storage operations.
|
||||
type Backend interface {
|
||||
Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error)
|
||||
}
|
||||
|
||||
// ArchiveItem describes one item to archive.
|
||||
type ArchiveItem struct {
|
||||
Kind string
|
||||
LocalPath string
|
||||
RemoteKey string
|
||||
}
|
||||
|
||||
// ArchiveRequest describes one archive operation.
|
||||
type ArchiveRequest struct {
|
||||
SessionID string
|
||||
ManifestPath string
|
||||
Items []ArchiveItem
|
||||
}
|
||||
|
||||
// ArchiveResult describes archive operation output.
|
||||
type ArchiveResult struct {
|
||||
Archived []ArchiveItem
|
||||
Metadata map[string]any
|
||||
}
|
||||
@@ -10,25 +10,11 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// NoopBackend is a deterministic no-op archive/storage adapter.
|
||||
type NoopBackend struct{}
|
||||
|
||||
// Archive returns the requested items as archived with placeholder metadata.
|
||||
func (n *NoopBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ArchiveResult{}, err
|
||||
}
|
||||
return ArchiveResult{Archived: append([]ArchiveItem(nil), req.Items...), Metadata: map[string]any{"placeholder": true}}, nil
|
||||
}
|
||||
|
||||
// FakeBackend captures archive requests and returns deterministic responses.
|
||||
// FakeBackend provides a deterministic in-memory object store for tests.
|
||||
type FakeBackend struct {
|
||||
Requests []ArchiveRequest
|
||||
Err error
|
||||
Result ArchiveResult
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
@@ -43,23 +29,10 @@ type FakeUploadCall struct {
|
||||
Options UploadOptions
|
||||
}
|
||||
|
||||
// Archive records request and returns configured response.
|
||||
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ArchiveResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return ArchiveResult{}, f.Err
|
||||
}
|
||||
res := f.Result
|
||||
if res.Archived == nil {
|
||||
res.Archived = append([]ArchiveItem(nil), req.Items...)
|
||||
}
|
||||
if res.Metadata == nil {
|
||||
res.Metadata = map[string]any{"fake": true}
|
||||
}
|
||||
return res, nil
|
||||
// FakeDownloadCall captures one download invocation in call order.
|
||||
type FakeDownloadCall struct {
|
||||
Key string
|
||||
LocalPath string
|
||||
}
|
||||
|
||||
// FakeObject is a deterministic fake object-store record.
|
||||
@@ -130,6 +103,10 @@ func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error
|
||||
if !ok {
|
||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||
}
|
||||
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||
Key: normalizeObjectKey(key),
|
||||
LocalPath: localPath,
|
||||
})
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||
|
||||
@@ -9,30 +9,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeBackendCapturesRequestAndReturnsItems(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
req := ArchiveRequest{SessionID: "s1", Items: []ArchiveItem{{Kind: "artifact", LocalPath: "artifacts/log.md"}}}
|
||||
|
||||
res, err := fake.Archive(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 || fake.Requests[0].SessionID != "s1" {
|
||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
||||
}
|
||||
if len(res.Archived) != 1 {
|
||||
t.Fatalf("archived len = %d, want 1", len(res.Archived))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendError(t *testing.T) {
|
||||
fake := &FakeBackend{Err: errors.New("boom")}
|
||||
_, err := fake.Archive(context.Background(), ArchiveRequest{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendListPrefixFiltering(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
|
||||
@@ -56,7 +32,7 @@ func TestFakeBackendDownload(t *testing.T) {
|
||||
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
|
||||
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(dst)
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
|
||||
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
|
||||
//
|
||||
// Key invariant:
|
||||
// callers pass full bucket-relative object keys. Backend implementations do not
|
||||
|
||||
36
internal/adapters/storage/temp_download.go
Normal file
36
internal/adapters/storage/temp_download.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DownloadObjectToTemp downloads an object into a temporary file and returns
|
||||
// the cleaned local path.
|
||||
func DownloadObjectToTemp(ctx context.Context, store ObjectStore, key, pattern string) (string, error) {
|
||||
if store == nil {
|
||||
return "", fmt.Errorf("object store is required")
|
||||
}
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
return "", fmt.Errorf("temp file pattern is required")
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", pattern)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return "", fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
|
||||
if err := store.Download(ctx, key, path); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
69
internal/adapters/storage/temp_download_test.go
Normal file
69
internal/adapters/storage/temp_download_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||
store := &FakeBackend{}
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Remove(path) })
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if string(data) != "run-123\n" {
|
||||
t.Fatalf("downloaded data = %q, want %q", string(data), "run-123\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
||||
sentinel := errors.New("download failed")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
pattern := "narratio-test-fail-*.txt"
|
||||
before, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("Glob(before) error = %v", err)
|
||||
}
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||
}
|
||||
if strings.TrimSpace(path) != "" {
|
||||
t.Fatalf("DownloadObjectToTemp() path = %q, want empty on failure", path)
|
||||
}
|
||||
after, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("Glob(after) error = %v", err)
|
||||
}
|
||||
if len(after) != len(before) {
|
||||
t.Fatalf("temp file count changed after failed download: before=%d after=%d", len(before), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
||||
sentinel := errors.New("object missing")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
if err == nil {
|
||||
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||
}
|
||||
err = fmt.Errorf("download run pointer failed: %w", err)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("wrapped error does not preserve sentinel cause: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -145,7 +145,7 @@ func TestHTTPClientDoesNotRetryOnNonRetryableStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientInvalidJSONFailsAndDoesNotPromote(t *testing.T) {
|
||||
func TestHTTPClientInvalidJSONFailsAndDoesNotInstallOutput(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`not-json`))
|
||||
}))
|
||||
|
||||
66
internal/app/analyze_artifacts.go
Normal file
66
internal/app/analyze_artifacts.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type artifactSelectionFlag struct {
|
||||
values []string
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) String() string {
|
||||
return strings.Join(f.values, ",")
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) Set(value string) error {
|
||||
f.values = append(f.values, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) Normalize() ([]string, error) {
|
||||
if len(f.values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(f.values))
|
||||
for _, raw := range f.values {
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
name := strings.TrimSpace(part)
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("artifact names must be non-empty")
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
||||
if len(selected) == 0 {
|
||||
return nil
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||
}
|
||||
configured := cfg.Pipeline.Scriptorium.Artifacts
|
||||
if len(configured) == 0 {
|
||||
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||
}
|
||||
for _, name := range selected {
|
||||
if _, ok := configured[name]; !ok {
|
||||
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
435
internal/app/analyze_artifacts_commands_test.go
Normal file
435
internal/app/analyze_artifacts_commands_test.go
Normal file
@@ -0,0 +1,435 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `run-stage: --artifacts is only supported for stages "analyze" and "publish"`) {
|
||||
t.Fatalf("stderr = %q, want stage-gating error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"publish"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{
|
||||
"run-stage", "publish", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--artifacts", "session_recap",
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `run: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
seed.MarkStageSucceeded("analyze", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stage=analyze executed=0 skipped=1 force=false") {
|
||||
t.Fatalf("output = %q, want analyze skip without force", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(
|
||||
context.Background(),
|
||||
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedForce = opts.Force
|
||||
return &RunSummary{
|
||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||
Executed: []string{"analyze"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
|
||||
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
|
||||
}
|
||||
if !capturedForce {
|
||||
t.Fatal("captured force = false, want true")
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
|
||||
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{
|
||||
"analyze",
|
||||
"2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--artifacts", "player_handout,session_recap",
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "extra positional", args: []string{"analyze", "2026-05-03", "extra"}, want: "analyze: unexpected positional arguments"},
|
||||
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedForce = opts.Force
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{
|
||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||
Executed: []string{"publish"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||
}
|
||||
if !capturedForce {
|
||||
t.Fatal("captured force = false, want true")
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio publish: executed=1 skipped=0 force=true; manifest=") {
|
||||
t.Fatalf("stdout = %q, want publish summary", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishRejectsUnsupportedArgsAndFlags(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "extra positional", args: []string{"publish", "2026-05-03", "extra"}, want: "publish: unexpected positional arguments"},
|
||||
{name: "force flag", args: []string{"publish", "--force"}, want: "publish: invalid flags: flag provided but not defined: -force"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `publish: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsageIncludesAnalyzeAndPublish(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(nil, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze") {
|
||||
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "publish") {
|
||||
t.Fatalf("stderr = %q, want usage to include publish", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("open pipeline config for append: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
extra := `
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
player_handout:
|
||||
enabled: true
|
||||
prompt_id: dnd.player_handout
|
||||
output_path: artifacts/player_handout.md
|
||||
depends_on:
|
||||
- session_recap
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
`
|
||||
if _, err := f.WriteString(extra); err != nil {
|
||||
t.Fatalf("append scriptorium config: %v", err)
|
||||
}
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
132
internal/app/analyze_artifacts_test.go
Normal file
132
internal/app/analyze_artifacts_test.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestArtifactSelectionFlagNormalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputs []string
|
||||
want []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "single value",
|
||||
inputs: []string{"session_recap"},
|
||||
want: []string{"session_recap"},
|
||||
},
|
||||
{
|
||||
name: "repeatable and comma separated values are deduped and sorted",
|
||||
inputs: []string{"session_recap,player_handout", "session_recap"},
|
||||
want: []string{"player_handout", "session_recap"},
|
||||
},
|
||||
{
|
||||
name: "empty token fails",
|
||||
inputs: []string{"session_recap,"},
|
||||
wantErr: "artifact names must be non-empty",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var flag artifactSelectionFlag
|
||||
for _, in := range tt.inputs {
|
||||
if err := flag.Set(in); err != nil {
|
||||
t.Fatalf("Set(%q) error = %v", in, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := flag.Normalize()
|
||||
if tt.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if err.Error() != tt.wantErr {
|
||||
t.Fatalf("Normalize() error = %q, want %q", err.Error(), tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("Normalize() len = %d, want %d; got=%v", len(got), len(tt.want), got)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("Normalize()[%d] = %q, want %q", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSelectedArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *config.Config
|
||||
selected []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "empty selection is accepted",
|
||||
cfg: &config.Config{},
|
||||
selected: nil,
|
||||
},
|
||||
{
|
||||
name: "scriptorium required when selected artifacts present",
|
||||
cfg: &config.Config{Pipeline: &config.PipelineConfig{}},
|
||||
selected: []string{"session_recap"},
|
||||
wantErr: "--artifacts requires pipeline.scriptorium.artifacts to be configured",
|
||||
},
|
||||
{
|
||||
name: "unknown selected artifact fails",
|
||||
cfg: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Scriptorium: &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
selected: []string{"player_handout"},
|
||||
wantErr: `--artifacts includes unknown artifact "player_handout"`,
|
||||
},
|
||||
{
|
||||
name: "known selected artifacts are accepted",
|
||||
cfg: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Scriptorium: &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||
"player_handout": {Enabled: true, PromptID: "dnd.player_handout", OutputPath: "artifacts/player_handout.md"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
selected: []string{"player_handout", "session_recap"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateSelectedArtifacts(tt.cfg, tt.selected)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if err.Error() != tt.wantErr {
|
||||
t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("error = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
44
internal/app/campaign_config_path.go
Normal file
44
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
||||
campaignID := strings.TrimSpace(campaignIDFlag)
|
||||
campaignFile := strings.TrimSpace(campaignFileFlag)
|
||||
if campaignID != "" && campaignFile != "" {
|
||||
return "", fmt.Errorf("--campaign and --campaign-file are mutually exclusive")
|
||||
}
|
||||
if campaignFile != "" {
|
||||
return filepath.Clean(campaignFile), nil
|
||||
}
|
||||
if campaignID == "" && pipelineCfg != nil {
|
||||
campaignID = strings.TrimSpace(pipelineCfg.Campaigns.DefaultCampaignID)
|
||||
}
|
||||
if campaignID == "" {
|
||||
return "", fmt.Errorf("no campaign selected; pass --campaign <id> or set pipeline.campaigns.default_campaign_id")
|
||||
}
|
||||
if err := validateCampaignIDToken(campaignID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if pipelineCfg == nil || strings.TrimSpace(pipelineCfg.Campaigns.Root) == "" {
|
||||
return "", fmt.Errorf("pipeline.campaigns.root is required to select campaign %q", campaignID)
|
||||
}
|
||||
return filepath.Clean(filepath.Join(pipelineCfg.Campaigns.Root, campaignID, "campaign.yml")), nil
|
||||
}
|
||||
|
||||
func validateCampaignIDToken(campaignID string) error {
|
||||
if filepath.IsAbs(campaignID) ||
|
||||
strings.Contains(campaignID, "/") ||
|
||||
strings.Contains(campaignID, `\`) ||
|
||||
campaignID == "." ||
|
||||
campaignID == ".." {
|
||||
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
84
internal/app/campaign_config_path_test.go
Normal file
84
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestResolveCampaignConfigPathCampaignFileWins(t *testing.T) {
|
||||
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||
got, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", explicit)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
if got != explicit {
|
||||
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathUsesSelectedCampaignID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = dir
|
||||
|
||||
got, err := resolveCampaignConfigPath(pipelineCfg, "icewind", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
want := filepath.Join(dir, "icewind", "campaign.yml")
|
||||
if got != filepath.Clean(want) {
|
||||
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathUsesDefaultCampaignID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = dir
|
||||
pipelineCfg.Campaigns.DefaultCampaignID = "dilfs"
|
||||
|
||||
got, err := resolveCampaignConfigPath(pipelineCfg, "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
want := filepath.Join(dir, "dilfs", "campaign.yml")
|
||||
if got != filepath.Clean(want) {
|
||||
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRejectsCampaignIDAndFile(t *testing.T) {
|
||||
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "dilfs", filepath.Join(t.TempDir(), "campaign.yml"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("error = %q, want mutual exclusion", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRequiresCampaignSelection(t *testing.T) {
|
||||
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no campaign selected") {
|
||||
t.Fatalf("error = %q, want missing selection guidance", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRejectsPathLikeCampaignID(t *testing.T) {
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = t.TempDir()
|
||||
|
||||
_, err := resolveCampaignConfigPath(pipelineCfg, "../icewind", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "single path segment") {
|
||||
t.Fatalf("error = %q, want path segment guidance", err.Error())
|
||||
}
|
||||
}
|
||||
281
internal/app/clean.go
Normal file
281
internal/app/clean.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
// state unless cache cleanup is explicitly requested.
|
||||
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var all bool
|
||||
var dryRun bool
|
||||
var clearCache bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
||||
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
||||
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
||||
if err := parseSessionAwareFlags("clean", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if all {
|
||||
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||
}
|
||||
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||
}
|
||||
|
||||
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||
}
|
||||
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||
if campaign == "" || sessionID == "" {
|
||||
return fmt.Errorf("clean: campaign and session_id are required")
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
||||
}
|
||||
|
||||
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
||||
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
||||
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||
strings.TrimSpace(flags.sessionID) != "" ||
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintln(out, "Clean plan for all local sessions")
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cleaned all local sessions")
|
||||
}
|
||||
|
||||
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
||||
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dryRun {
|
||||
if dir.Exists {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !dir.Exists {
|
||||
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
||||
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(entry); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||
rootAbs, exists, err := validateCleanRoot(root, policy)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if !exists {
|
||||
return rootAbs, nil, nil
|
||||
}
|
||||
entries, err := os.ReadDir(rootAbs)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
out := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(rootAbs, entry.Name())
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
||||
}
|
||||
out = append(out, path)
|
||||
}
|
||||
return rootAbs, out, nil
|
||||
}
|
||||
|
||||
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||
if cfg.Session.Inputs.AudioS3 == nil {
|
||||
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||
return nil
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := store.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
count := 0
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||
continue
|
||||
}
|
||||
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if deleted {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
||||
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
||||
}
|
||||
|
||||
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
||||
file, err := validateScopedFile(root, target, policy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if dryRun {
|
||||
if file.Exists {
|
||||
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if !file.Exists {
|
||||
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if err := os.Remove(file.TargetAbs); err != nil {
|
||||
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, false)
|
||||
}
|
||||
|
||||
func cleanIsFlac(path string) bool {
|
||||
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||
}
|
||||
255
internal/app/clean_test.go
Normal file
255
internal/app/clean_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workDir)
|
||||
cleanAssertMissing(t, spoolDir)
|
||||
cleanAssertExists(t, cachePath)
|
||||
if !strings.Contains(stdout.String(), "Cache: preserved") {
|
||||
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--dry-run"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertExists(t, workDir)
|
||||
cleanAssertExists(t, spoolDir)
|
||||
if !strings.Contains(stdout.String(), "Would delete:") {
|
||||
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Missing:") {
|
||||
t.Fatalf("stdout = %q, want missing path output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write session: %v", err)
|
||||
}
|
||||
|
||||
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, cachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
|
||||
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
workRoot := filepath.Join(workspaceRoot, "work")
|
||||
spoolRoot := filepath.Join(workspaceRoot, "spool")
|
||||
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workRoot)
|
||||
cleanAssertExists(t, spoolRoot)
|
||||
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
|
||||
cleanAssertExists(t, cachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, audioCachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, audioCachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--all"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--all cannot be combined") {
|
||||
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRequiresSessionID(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "session_id is required unless --all is set") {
|
||||
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRejectsUnsafeTargets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
|
||||
t.Fatal("outside target error = nil, want error")
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
|
||||
t.Fatal("root target error = nil, want error")
|
||||
}
|
||||
filePath := filepath.Join(root, "file.txt")
|
||||
mustWriteTestFile(t, filePath, "file")
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
|
||||
t.Fatal("file target error = nil, want error")
|
||||
}
|
||||
symlinkPath := filepath.Join(root, "link")
|
||||
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
|
||||
t.Fatal("symlink target error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearIsNotCommandAlias(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clear"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
|
||||
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected %q to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
82
internal/app/cleanup_targets.go
Normal file
82
internal/app/cleanup_targets.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func validateScopedTarget(root, target, policy string, requireDir bool) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
if cleanRoot == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
if cleanTarget == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||
}
|
||||
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
targetAbs, err := filepath.Abs(cleanTarget)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||
}
|
||||
if rel == "." {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||
}
|
||||
|
||||
info, err := os.Lstat(targetAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||
}
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||
}
|
||||
if requireDir && !info.IsDir() {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||
}
|
||||
if !requireDir && info.IsDir() {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
|
||||
}
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||
}
|
||||
|
||||
func validateCleanRoot(root, policy string) (string, bool, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
if cleanRoot == "" {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
info, err := os.Lstat(rootAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return rootAbs, false, nil
|
||||
}
|
||||
return "", false, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
|
||||
}
|
||||
return rootAbs, true, nil
|
||||
}
|
||||
103
internal/app/cleanup_targets_test.go
Normal file
103
internal/app/cleanup_targets_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanValidateScopedDirAndFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dirTarget := filepath.Join(root, "runs", "run-1")
|
||||
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||
if err := os.MkdirAll(dirTarget, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(dirTarget) error = %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := validateScopedDir(root, dirTarget, "test.dir"); err != nil {
|
||||
t.Fatalf("validateScopedDir() error = %v", err)
|
||||
}
|
||||
if _, err := validateScopedFile(root, fileTarget, "test.file"); err != nil {
|
||||
t.Fatalf("validateScopedFile() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanValidateScopedTargetSafetyRules(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
target := filepath.Join(root, "runs", "run-1")
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(target) error = %v", err)
|
||||
}
|
||||
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||
}
|
||||
symlinkTarget := filepath.Join(root, "symlink")
|
||||
if err := os.Symlink(target, symlinkTarget); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := validateScopedDir(root, root, "test.root"); err == nil || !strings.Contains(err.Error(), "refusing to delete root directory") {
|
||||
t.Fatalf("validateScopedDir(root) error = %v, want root deletion rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, filepath.Join(outside, "x"), "test.outside"); err == nil || !strings.Contains(err.Error(), "outside root") {
|
||||
t.Fatalf("validateScopedDir(outside) error = %v, want outside-root rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, fileTarget, "test.file-as-dir"); err == nil || !strings.Contains(err.Error(), "is not a directory") {
|
||||
t.Fatalf("validateScopedDir(file) error = %v, want not-a-directory rejection", err)
|
||||
}
|
||||
if _, err := validateScopedFile(root, target, "test.dir-as-file"); err == nil || !strings.Contains(err.Error(), "is a directory") {
|
||||
t.Fatalf("validateScopedFile(dir) error = %v, want is-a-directory rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, symlinkTarget, "test.symlink"); err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||
t.Fatalf("validateScopedDir(symlink) error = %v, want symlink rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanableRootChildrenRejectsSymlinkChild(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
realChild := filepath.Join(root, "runs")
|
||||
if err := os.MkdirAll(realChild, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(realChild) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(realChild, filepath.Join(root, "link")); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
|
||||
_, _, err := cleanableRootChildren(root, "test.root.children")
|
||||
if err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||
t.Fatalf("cleanableRootChildren() error = %v, want symlink rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanValidateScopedTargetMissing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
missingDir := filepath.Join(root, "runs", "missing")
|
||||
got, err := validateScopedDir(root, missingDir, "test.missing")
|
||||
if err != nil {
|
||||
t.Fatalf("validateScopedDir(missing) error = %v", err)
|
||||
}
|
||||
if got.Exists {
|
||||
t.Fatalf("validateScopedDir(missing).Exists = true, want false")
|
||||
}
|
||||
|
||||
missingFile := filepath.Join(root, "cache", "missing.flac")
|
||||
got, err = validateScopedFile(root, missingFile, "test.missing.file")
|
||||
if err != nil {
|
||||
t.Fatalf("validateScopedFile(missing) error = %v", err)
|
||||
}
|
||||
if got.Exists {
|
||||
t.Fatalf("validateScopedFile(missing).Exists = true, want false")
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage"}
|
||||
var supportedCommands = []string{"run", "run-stage", "analyze", "publish", "clean", "session"}
|
||||
|
||||
// Execute dispatches CLI commands and returns a process exit code.
|
||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
@@ -24,14 +24,16 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
switch cmd {
|
||||
case "run":
|
||||
err = Run(ctx, cmdArgs, stdout)
|
||||
case "plan":
|
||||
err = Plan(ctx, cmdArgs, stdout)
|
||||
case "status":
|
||||
err = Status(ctx, cmdArgs, stdout)
|
||||
case "resume":
|
||||
err = Resume(ctx, cmdArgs, stdout)
|
||||
case "run-stage":
|
||||
err = RunStage(ctx, cmdArgs, stdout)
|
||||
case "analyze":
|
||||
err = Analyze(ctx, cmdArgs, stdout)
|
||||
case "publish":
|
||||
err = Publish(ctx, cmdArgs, stdout)
|
||||
case "session":
|
||||
err = Session(ctx, cmdArgs, stdout)
|
||||
case "clean":
|
||||
err = Clean(ctx, cmdArgs, stdout)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||
printUsage(stderr)
|
||||
@@ -49,8 +51,3 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
func printUsage(w io.Writer) {
|
||||
fmt.Fprintf(w, "Usage: narratio <%s>\n", strings.Join(supportedCommands, "|"))
|
||||
}
|
||||
|
||||
func placeholder(out io.Writer, command string) error {
|
||||
_, err := fmt.Fprintf(out, "narratio %s: not yet implemented\n", command)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -24,19 +24,17 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
manifestPath := writeManifestPathForExecute(t)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nextract: run\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -64,13 +62,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "run missing flags", args: []string{"run"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "plan missing flags", args: []string{"plan"}, want: "plan: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
|
||||
{name: "resume missing flags", args: []string{"resume"}, want: "resume: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
|
||||
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "run missing config uses defaults", args: []string{"run", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
||||
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
||||
{name: "status old top-level removed", args: []string{"status"}, want: `unknown command: "status"`},
|
||||
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
|
||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -94,12 +92,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
|
||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "unknown", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -108,16 +106,32 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
func TestExecuteRunStageArchiveAliasFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown stage "archive"`) {
|
||||
t.Fatalf("stderr = %q, want unknown stage alias error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "normalize", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -136,19 +150,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
|
||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||
code = Execute([]string{"run-stage", "transcribe", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -156,7 +170,7 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
t.Fatal("expected whisperx server to be called at least once")
|
||||
}
|
||||
|
||||
outPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||
outPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||
data, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
||||
@@ -188,6 +202,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
@@ -202,8 +217,6 @@ seriatim:
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
llm_api_key_env: OPENROUTER_API_KEY
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
@@ -214,6 +227,8 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -233,13 +248,13 @@ inputs:
|
||||
_ = os.Chdir(originalWD)
|
||||
})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", sessionID)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "polish", sessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -252,6 +267,7 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
@@ -266,8 +282,6 @@ seriatim:
|
||||
binary: seriatim
|
||||
audita:
|
||||
binary: audita
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
@@ -278,6 +292,8 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -288,7 +304,7 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -305,24 +321,109 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||
defer func() {
|
||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||
}()
|
||||
_ = campaignPath
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"run", "2026-05-03", "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=9 skipped=0; manifest=") {
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMissingCampaignConfigReportsRegistryPath(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
if err := os.Remove(campaignPath); err != nil {
|
||||
t.Fatalf("remove campaign config: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "load campaign config") {
|
||||
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), filepath.ToSlash(filepath.Join("campaigns", "sample-campaign", "campaign.yml"))) {
|
||||
t.Fatalf("stderr = %q, want campaign registry path", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsesPipelineDefaultCampaignID(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Campaign: sample-campaign") {
|
||||
t.Fatalf("stdout = %q, want default campaign", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCampaignIDSelectsRegistryCampaign(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
campaignRoot := filepath.Dir(filepath.Dir(campaignPath))
|
||||
otherDir := filepath.Join(campaignRoot, "icewind")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "campaign.yml"), `campaign_id: icewind
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "icewind", "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Campaign: icewind") {
|
||||
t.Fatalf("stdout = %q, want selected campaign", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsCampaignIDAndCampaignFile(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "sample-campaign", "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "mutually exclusive") {
|
||||
t.Fatalf("stderr = %q, want mutually exclusive error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteInvalidCommand(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -359,29 +460,42 @@ func TestExecuteMissingCommand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignRoot := filepath.Join(dir, "campaigns")
|
||||
campaignDir := filepath.Join(campaignRoot, "sample-campaign")
|
||||
campaignPath := filepath.Join(campaignDir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
url := "https://example.com/transcribe"
|
||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||
url = transcribeURL[0]
|
||||
}
|
||||
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||
scriptoriumBinary := writeScriptoriumAppTestWrapper(t)
|
||||
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||
t.Setenv("GO_WANT_APP_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
||||
t.Setenv("PATH", filepath.Dir(scriptoriumBinary)+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
campaigns:
|
||||
root: ` + campaignRoot + `
|
||||
default_campaign_id: sample-campaign
|
||||
cache:
|
||||
root: ` + filepath.Join(workspaceRoot, "cache") + `
|
||||
spool:
|
||||
root: ` + filepath.Join(workspaceRoot, "spool") + `
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: test-bucket
|
||||
archive:
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: false
|
||||
whisperx:
|
||||
@@ -398,36 +512,61 @@ seriatim:
|
||||
report: true
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
artifacts:
|
||||
output_dir: artifacts
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`
|
||||
campaignYAML := `campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline config: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(campaignDir, 0o755); err != nil {
|
||||
t.Fatalf("create campaign dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign config: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session config: %v", err)
|
||||
}
|
||||
|
||||
mustWriteTestFile(t, filepath.Join(dir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, sessionPath
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
campaignYAML := `campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
return campaignPath
|
||||
}
|
||||
|
||||
func writeManifestPathForExecute(t *testing.T) string {
|
||||
@@ -469,6 +608,60 @@ func writeSeriatimAppTestWrapper(t *testing.T) string {
|
||||
return path
|
||||
}
|
||||
|
||||
func writeScriptoriumAppTestWrapper(t *testing.T) string {
|
||||
t.Helper()
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "scriptorium")
|
||||
content := "#!/bin/sh\nexec \"" + exe + "\" -test.run=TestScriptoriumAppHelper -- \"$@\"\n"
|
||||
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestScriptoriumAppHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_APP_SCRIPTORIUM_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
args := os.Args
|
||||
start := -1
|
||||
for i := range args {
|
||||
if args[i] == "--" {
|
||||
start = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 || start >= len(args) {
|
||||
_, _ = os.Stderr.WriteString("missing -- args separator\n")
|
||||
os.Exit(2)
|
||||
}
|
||||
runArgs := args[start:]
|
||||
|
||||
outputPath := appSeriatimFlagValue(runArgs, "--out")
|
||||
if strings.TrimSpace(outputPath) == "" {
|
||||
outputPath = appSeriatimFlagValue(runArgs, "--output")
|
||||
}
|
||||
if strings.TrimSpace(outputPath) == "" {
|
||||
_, _ = os.Stderr.WriteString("missing output flag\n")
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
_, _ = os.Stderr.WriteString(fmt.Sprintf("mkdir output dir: %v\n", err))
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := os.WriteFile(outputPath, []byte(`{"trim_action":"copy","warnings":[]}`), 0o644); err != nil {
|
||||
_, _ = os.Stderr.WriteString(fmt.Sprintf("write output: %v\n", err))
|
||||
os.Exit(2)
|
||||
}
|
||||
_, _ = os.Stdout.WriteString("scriptorium helper stdout\n")
|
||||
_, _ = os.Stderr.WriteString("scriptorium helper stderr\n")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func TestSeriatimAppHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_APP_SERIATIM_HELPER") != "1" {
|
||||
return
|
||||
|
||||
141
internal/app/config_loader.go
Normal file
141
internal/app/config_loader.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type pipelineCampaignConfig struct {
|
||||
PipelinePath string
|
||||
CampaignPath string
|
||||
Pipeline *config.PipelineConfig
|
||||
Campaign *config.CampaignConfig
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||
}
|
||||
|
||||
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if discoveredSession.Path != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||
if sessionID == "" {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
partialCfg := &config.Config{
|
||||
Pipeline: base.Pipeline,
|
||||
Campaign: base.Campaign,
|
||||
PipelinePath: base.PipelinePath,
|
||||
CampaignPath: base.CampaignPath,
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
|
||||
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := storage.DownloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config.Resolve(
|
||||
base.PipelinePath,
|
||||
base.Pipeline,
|
||||
base.CampaignPath,
|
||||
base.Campaign,
|
||||
sessionTempPath,
|
||||
sessionCfg,
|
||||
config.SessionSource{
|
||||
Source: "session_config.s3",
|
||||
LocalPath: sessionTempPath,
|
||||
S3Bucket: s3BucketName(base.Pipeline),
|
||||
S3Key: remoteKey,
|
||||
S3Size: sessionInfo.Size,
|
||||
S3ETag: sessionInfo.ETag,
|
||||
SpoolPath: sessionTempPath,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvedCampaignPath, err := resolveCampaignConfigPath(pipelineCfg, campaignFlag, campaignFileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if selectedID := strings.TrimSpace(campaignFlag); selectedID != "" && strings.TrimSpace(campaignFileFlag) == "" {
|
||||
if got := config.CampaignID(campaignCfg); got != selectedID {
|
||||
return nil, fmt.Errorf("campaign config %q invalid: campaign_id %q does not match selected campaign %q", resolvedCampaignPath, got, selectedID)
|
||||
}
|
||||
}
|
||||
return &pipelineCampaignConfig{
|
||||
PipelinePath: resolvedPipelinePath,
|
||||
CampaignPath: resolvedCampaignPath,
|
||||
Pipeline: pipelineCfg,
|
||||
Campaign: campaignCfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||
objects, err := store.List(ctx, sessionPrefix)
|
||||
if err != nil {
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == remoteKey {
|
||||
return obj, nil
|
||||
}
|
||||
}
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||
}
|
||||
|
||||
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||
if cfg == nil || cfg.Storage.S3 == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||
}
|
||||
416
internal/app/extract_lifecycle_test.go
Normal file
416
internal/app/extract_lifecycle_test.go
Normal file
@@ -0,0 +1,416 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type materializingNotariusRunner struct {
|
||||
cfg *config.NotariusConfig
|
||||
requests []notarius.RunRequest
|
||||
failuresRemaining int
|
||||
}
|
||||
|
||||
func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunRequest) (notarius.RunResult, error) {
|
||||
r.requests = append(r.requests, req)
|
||||
if r.failuresRemaining > 0 {
|
||||
r.failuresRemaining--
|
||||
return notarius.RunResult{}, errors.New("notarius execution failed")
|
||||
}
|
||||
externalRunID := fmt.Sprintf("notarius-run-%d", len(r.requests))
|
||||
bundle := filepath.Join(req.OutputRoot, externalRunID)
|
||||
lanesDir := filepath.Join(bundle, "lanes")
|
||||
if err := os.MkdirAll(lanesDir, 0o755); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
for path, content := range map[string]string{
|
||||
filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`,
|
||||
filepath.Join(bundle, "manifest.json"): `{}`,
|
||||
filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`,
|
||||
filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`,
|
||||
filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`,
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
}
|
||||
output := r.cfg.Outputs["npc_registry"]
|
||||
return notarius.RunResult{
|
||||
Receipt: notarius.Receipt{
|
||||
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID,
|
||||
PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json",
|
||||
NormalizedOutputCount: 1, ValidationStatus: "valid",
|
||||
},
|
||||
BundleRoot: bundle,
|
||||
Index: notarius.Index{
|
||||
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||
Lanes: []notarius.LaneDescriptor{{
|
||||
LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"),
|
||||
MediaType: output.MediaType, SchemaID: output.SchemaID,
|
||||
SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||
}},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestExtractLifecycleDisabledThenEnabled(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(first.Executed) != 1 || len(first.Skipped) != 1 || first.Skipped[0] != "extract" || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled summary = %#v requests=%d", first, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 0 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled summary = %#v requests=%d", second, len(runner.requests))
|
||||
}
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), second.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSucceeded || len(loaded.Stages["extract"].Outputs) != 2 {
|
||||
t.Fatalf("extract record = %#v, want succeeded manifest-ready outputs", loaded.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleChangedOutcomeRerunsSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 2 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled run analyze=%d Notarius=%d, want 2 and 1", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||
t.Fatalf("enabled summary = %#v, want extract and analyze executed", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
runner.failuresRemaining = 1
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "notarius execution failed") {
|
||||
t.Fatalf("failed executeStages() error = %v", err)
|
||||
}
|
||||
failed := loadLifecycleManifest(t, cfg)
|
||||
if failed.Stages["extract"].Status != manifest.StatusFailed || failed.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("failed lifecycle extract=%#v analyze=%#v", failed.Stages["extract"], failed.Stages["analyze"])
|
||||
}
|
||||
if failed.Stages["analyze"].Error == nil || failed.Stages["analyze"].Error.Message != staleReasonFailure {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", failed.Stages["analyze"].Error, staleReasonFailure)
|
||||
}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("retry executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 2 || len(summary.Executed) != 2 {
|
||||
t.Fatalf("retry analyze=%d Notarius=%d summary=%#v", analyzeRuns, len(runner.requests), summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusSkipped || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced self-skip extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
succeeded := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||
if succeeded == nil || succeeded.Status != manifest.StatusSucceeded || len(succeeded.Outputs) == 0 || len(succeeded.Logs) == 0 || len(succeeded.Metadata) == 0 {
|
||||
t.Fatalf("initial extraction result = %#v, want succeeded result details", succeeded)
|
||||
}
|
||||
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
runner.failuresRemaining = 1
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err == nil {
|
||||
t.Fatal("executeStages() error = nil, want forced extraction failure")
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusFailed || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced failure extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
failed := loaded.Stages["extract"]
|
||||
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||
}
|
||||
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||
}
|
||||
historicalExtract := historical.Stages["extract"]
|
||||
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||
}
|
||||
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleRepeatedSelfSkipPreservesSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("second executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("repeated disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 2 {
|
||||
t.Fatalf("second summary = %#v, want executed self-skip and skipped analyze", second)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("analyze = %#v, want succeeded", loaded.Stages["analyze"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("resume executeStages() error = %v", err)
|
||||
}
|
||||
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("resume summary = %#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleResumesAndRerunsObsoleteResults(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *config.Config, *manifest.Manifest)
|
||||
}{
|
||||
{name: "configuration changed", mutate: func(_ *testing.T, cfg *config.Config, _ *manifest.Manifest) {
|
||||
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||
output.SchemaVersion = "v2"
|
||||
cfg.Pipeline.Notarius.Outputs["npc_registry"] = output
|
||||
}},
|
||||
{name: "payload missing", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("Remove() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "payload tampered", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"npcs":["tampered"]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "record incompatible", mutate: func(_ *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaID = "incompatible"
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
persisted, err := (&manifest.LocalStore{}).Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
test.mutate(t, cfg, persisted)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save(mutated) error = %v", err)
|
||||
}
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun executeStages() error = %v", err)
|
||||
}
|
||||
if len(rerun.Executed) != 1 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 {
|
||||
t.Fatalf("rerun summary = %#v requests=%d", rerun, len(runner.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleUnsafeResumeErrorPreservesSuccess(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
persisted, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
persisted.Stages["extract"].Outputs[0].LocalPath = filepath.Join(cfg.Pipeline.Workspace.Root, "outside.json")
|
||||
if err := store.Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
before, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": persisted.Stages["extract"],
|
||||
"analyze": persisted.Stages["analyze"],
|
||||
})
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||
t.Fatalf("executeStages() error = %v, want unsafe resume failure", err)
|
||||
}
|
||||
afterManifest, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(after) error = %v", err)
|
||||
}
|
||||
after, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": afterManifest.Stages["extract"],
|
||||
"analyze": afterManifest.Stages["analyze"],
|
||||
})
|
||||
if string(before) != string(after) || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("successful records changed: before=%s after=%s requests=%d analyze=%d", before, after, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecyclePlan(t *testing.T, analyzeRuns *int) []stage.Stage {
|
||||
t.Helper()
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
return append(plan, countingStage{name: "analyze", runs: analyzeRuns})
|
||||
}
|
||||
|
||||
func loadLifecycleManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
|
||||
func markLifecycleStageSucceeded(t *testing.T, cfg *config.Config, name string) {
|
||||
t.Helper()
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
loaded.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), loaded); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *stage.Env, *materializingNotariusRunner) {
|
||||
t.Helper()
|
||||
cfg := testConfig(t)
|
||||
root := cfg.Pipeline.Workspace.Root
|
||||
binary := filepath.Join(root, "notarius")
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
workingDirectory := filepath.Join(root, "notarius-work")
|
||||
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||
}
|
||||
cfg.Pipeline.Notarius = &config.NotariusConfig{
|
||||
Enabled: enabled, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"npc_registry": {
|
||||
LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||
},
|
||||
},
|
||||
}
|
||||
paths, err := artifacts.NewLocalStore(root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||
}
|
||||
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||
if err := os.WriteFile(inputPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
LocalPath: inputPath,
|
||||
}})
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatalf("Save(seed) error = %v", err)
|
||||
}
|
||||
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||
return cfg, &stage.Env{Notarius: runner}, runner
|
||||
}
|
||||
21
internal/app/object_store.go
Normal file
21
internal/app/object_store.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||
}
|
||||
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
165
internal/app/object_store_test.go
Normal file
165
internal/app/object_store_test.go
Normal file
@@ -0,0 +1,165 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
fake := &storage.FakeBackend{}
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||
return nil, errors.New("access key was not loaded before object store init")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||
return nil, errors.New("secret key was not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
if store != fake {
|
||||
t.Fatalf("store = %#v, want fake backend", store)
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("object store factory was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||
t.Setenv(secretKeyEnv, "existing-secret")
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||
return nil, errors.New("existing access key was overwritten")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||
return nil, errors.New("existing secret key was overwritten")
|
||||
}
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("object store factory was called after secret load failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig("")
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("factory boom")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
|
||||
t.Fatalf("error = %q, want factory context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{
|
||||
Backend: "s3",
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.TrimSpace(secretsDir) != "" {
|
||||
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||
t.Helper()
|
||||
originals := make(map[string]string, len(names))
|
||||
present := make(map[string]bool, len(names))
|
||||
for _, name := range names {
|
||||
value, ok := os.LookupEnv(name)
|
||||
originals[name] = value
|
||||
present[name] = ok
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, name := range names {
|
||||
if present[name] {
|
||||
_ = os.Setenv(name, originals[name])
|
||||
} else {
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user