115 Commits

Author SHA1 Message Date
e433c86203 Close out the completed audit 2026-08-11 12:53:31 +00:00
a2a144dffa Close diagnostic and restore coverage gaps 2026-08-11 12:28:55 +00:00
8ef6e99d69 Bound remote control object reads 2026-08-11 03:43:18 +00:00
2545faef6c Dispose subprocess descendants after leader exit 2026-08-11 03:22:21 +00:00
80be8be4d6 Confine subprocess diagnostics and retain redacted tails 2026-08-11 03:11:27 +00:00
801adb385d Reconcile lifecycle documentation 2026-08-10 23:18:29 +00:00
feba7b9d74 Enforce continuous validation 2026-08-10 23:08:52 +00:00
b89224bbde Simplify stage and Audita contracts 2026-08-10 22:43:22 +00:00
131ffd9887 Make analyze input resolution deterministic 2026-08-10 22:36:21 +00:00
af492c9e97 Centralize effective artifact selection 2026-08-10 22:27:06 +00:00
f39fc94610 Bind extraction reuse to transcript identity 2026-08-10 22:15:53 +00:00
4e4eff6ba7 Centralize extraction bundle evidence 2026-08-10 22:06:48 +00:00
8ff1b4fa66 Enforce requested adapter output paths 2026-08-10 21:57:54 +00:00
702f622e18 Harden prepare and transcribe transitions 2026-08-10 21:53:54 +00:00
9da2c1e144 Stream WhisperX uploads safely 2026-08-10 21:49:44 +00:00
32653f54f9 Make configuration truthful and clean remote session files 2026-08-10 21:41:00 +00:00
72a200968a Harden configuration validation 2026-08-10 21:32:15 +00:00
b39b68add7 Unify previous source resolution 2026-08-10 21:20:59 +00:00
d9fa1d9328 Bind audio cache reuse to remote identity 2026-08-10 21:09:21 +00:00
8375ad83f3 Serialize restore recovery and rebase manifest paths 2026-08-10 21:01:28 +00:00
4158394dcf Bind restore to committed remote snapshots 2026-08-10 20:42:43 +00:00
eac7e155a5 Persist retryable post-publish cleanup obligations 2026-08-10 20:29:00 +00:00
0cf2cbfeb3 Make remote publish locks generation-safe 2026-08-10 20:17:54 +00:00
361dbb4ca8 Publish immutable remote commits 2026-08-10 20:05:24 +00:00
d6deccf3e8 Add immutable remote commit reader 2026-08-10 19:47:12 +00:00
ee747243fe Terminalize handled invocation failures 2026-08-10 19:34:46 +00:00
a1ceb457e9 Unify invocation manifest identity 2026-08-10 19:25:03 +00:00
9900211fa4 Confine publish archive reads 2026-08-10 19:16:18 +00:00
60cebf0e4b Redact and cap subprocess diagnostics 2026-08-10 18:55:49 +00:00
7bd575187e Terminate owned subprocess trees 2026-08-10 18:44:10 +00:00
ab5a7e8e3d Bound external result file reads 2026-08-10 18:30:28 +00:00
99b2e1cd81 Harden API key file loading 2026-08-10 18:24:29 +00:00
363313d99c Confine cleanup and use held session locks 2026-08-10 18:14:09 +00:00
18ddf00d3d Confine local file installation paths 2026-08-10 17:59:29 +00:00
59f3fe3d1d Make atomic file replacement crash durable 2026-08-10 17:44:38 +00:00
1dccf5f140 Validate portable workspace identifiers 2026-08-10 17:35:52 +00:00
0b40cf8026 Make ordinary workspaces group shareable 2026-08-10 17:27:00 +00:00
a7ec195587 Prepare a staged implementation plan to address the audit findings. 2026-08-10 17:08:57 +00:00
13de820931 Finalize codebase audit findings 2026-08-10 15:14:54 +00:00
14ef59aaed Document test suite policy audit conclusions 2026-08-10 15:04:05 +00:00
f387222fce Document maintainability audit conclusions 2026-08-10 14:53:19 +00:00
9cb9008dfc Document analyze dependency audit findings 2026-08-10 14:38:49 +00:00
083decc5b4 Document extraction audit findings 2026-08-10 14:24:03 +00:00
57cac5d3f7 Document ordinary stage audit findings 2026-08-10 14:09:54 +00:00
0a772e03b4 Document external adapter audit findings 2026-08-10 13:53:49 +00:00
0920062a38 Document configuration and composition audit findings 2026-08-10 13:33:59 +00:00
39afe644eb Document restore and previous-state audit findings 2026-08-10 13:16:35 +00:00
e3ee3de10a Audit publish commit and cleanup behavior 2026-08-10 12:55:10 +00:00
9c72db56e9 Audit artifact paths and filesystem safety 2026-08-10 12:42:49 +00:00
bb2d606dbb Audit runner and manifest lifecycle behavior 2026-08-10 12:28:36 +00:00
9850767a8a Establish the codebase audit baseline and contract map 2026-08-10 12:12:48 +00:00
74e2d21de5 Close the completed roadmap documents 2026-08-10 03:24:49 +00:00
7cb18a1a40 Reconcile promotion and manifest documentation 2026-08-10 03:04:20 +00:00
b556fc2f4f Clear superseded session stage result details 2026-08-10 02:53:28 +00:00
b99bd38eb4 Harden bundle promotion against symlink replacement 2026-08-10 02:45:51 +00:00
701b6726d7 Reconcile Notarius extraction documentation 2026-08-10 02:10:37 +00:00
665039f4dc Support atomic directory promotion across platforms 2026-08-10 01:58:52 +00:00
ef8dae776e Enforce canonical Notarius bundle paths 2026-08-10 01:50:31 +00:00
d01775b68a Exclude staged Notarius bundles from publish uploads 2026-08-10 01:43:43 +00:00
0d6f2dd0ce Invalidate downstream results when stages are replaced 2026-08-10 01:38:24 +00:00
df40cbec6e Document and validate Notarius extraction workflows 2026-08-10 00:42:44 +00:00
0341e0c7c0 Publish and inspect configured extraction artifacts 2026-08-10 00:32:24 +00:00
39af7d4f3c Integrate extraction artifacts into analysis catalog 2026-08-10 00:24:02 +00:00
bba582b4ca Integrate extraction lifecycle and resume validation 2026-08-10 00:14:46 +00:00
1f16a85330 Implement direct Notarius extraction execution 2026-08-09 23:59:50 +00:00
f9482639d4 Add the Notarius subprocess adapter 2026-08-09 23:47:38 +00:00
dce721cdbd Add safe immutable directory promotion 2026-08-09 23:37:02 +00:00
98734644d6 Add Notarius configuration and extraction source policy 2026-08-09 23:30:32 +00:00
951383226c Add artifact provenance and stage skip outcomes 2026-08-09 23:20:32 +00:00
df58595d1e Remove completed documentation alignment roadmaps 2026-08-09 22:02:07 +00:00
c3c14e7468 Complete documentation alignment roadmap 2026-08-09 21:53:04 +00:00
e7319ea016 Align internal documentation and maintained examples 2026-08-09 21:50:21 +00:00
bd2d5e2496 Clarify user and integration documentation contracts 2026-08-09 21:41:23 +00:00
115a44f629 Align documentation entry points and internal overview 2026-08-09 21:32:11 +00:00
18411dc5b5 Move contributor guidance to its canonical location 2026-08-09 21:27:58 +00:00
e23dc1ab6e Refocus the Narratio architecture policy 2026-08-09 21:26:18 +00:00
e1359ea227 Adopt canonical documentation ownership policy 2026-08-09 21:23:50 +00:00
7fdd99ec27 Prepare roadmap for documentation policy update 2026-08-09 21:21:00 +00:00
a90231ce0c Implement support for passing a session_id variable to scriptorium to support sticky routing 2026-07-02 21:04:37 -05:00
ed879b8bb0 Clean up obsolete placeholder code 2026-07-02 20:47:08 -05:00
717451512a Implemented new campaign/session stable inputs and corresponding input source references
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-27 09:34:05 -05:00
3ddb3a947b Update pipeline defaults so trim is enabled when omitted 2026-05-27 08:35:02 -05:00
c6632d5576 Bugfix in the seriatim adapter
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-27 08:09:22 -05:00
ffc07922c7 Cleanup following the render stage implementation and remove the completed roadmap
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-25 08:35:18 -05:00
f3310d4d16 Finalize render documentation across operations, integrations, troubleshooting, and roadmap status 2026-05-25 00:48:15 +00:00
88cee96d8d Finish render rollout with markdown publish defaults, analyze guidance, and docs updates 2026-05-25 00:46:27 +00:00
2fece10215 Implement render stage runtime and integrate it into pipeline execution 2026-05-25 00:40:06 +00:00
0658f2f642 Add render artifact model, config, and Seriatim adapter contracts 2026-05-25 00:28:01 +00:00
a51228c803 Add a documentation roadmap for the upcoming render stage feature 2026-05-24 19:15:42 -05:00
4491fb5ccd Final documentation cleanup for v1.0.0 release
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-23 11:28:03 -05:00
30b905765c Remove the deprecated narratio resume command 2026-05-23 11:25:08 -05:00
03eac70881 Mark cleanup roadmap stages as implemented 2026-05-23 16:05:21 +00:00
0f7e6b979f Deduplicate locks add/remove session-id and source parsing 2026-05-23 16:03:20 +00:00
c366912586 Extract shared read-only session inspection checks 2026-05-23 16:00:31 +00:00
9fe44cd00d Centralize Scriptorium input source policy across config, analyze, and previous-cache 2026-05-23 15:50:29 +00:00
094b0d2532 Centralize path-safe root joins and atomic file operations 2026-05-23 15:45:31 +00:00
98649f4d81 Add a roadmap to implement the remaining items identfied by the code quality audit 2026-05-23 10:35:26 -05:00
8a559efd5b Audit code quality and deduplication opportunities 2026-05-23 10:10:21 -05:00
72deccb4e2 Implement final changes from the code quality and deduplication opportunity audit 2026-05-23 10:05:14 -05:00
5620fc5bcf Refresh CLI and internal restore documentation for current behavior 2026-05-23 14:07:11 +00:00
be57e675e0 Split operator helper implementations by command responsibility 2026-05-23 14:03:31 +00:00
3971443831 Centralize remote current-state loading and preserve caller policy 2026-05-23 13:56:53 +00:00
a6b0c33e9f Unify session-aware CLI parsing and add session-id compatibility 2026-05-23 13:47:39 +00:00
96b886e711 Align internal publish terminology across stage, app, and artifacts 2026-05-23 13:39:15 +00:00
7d584ee6cd Centralize artifact source and publish destination policy 2026-05-23 13:28:25 +00:00
572a112c31 Consolidate path safety, temp downloads, and cleanup validation helpers 2026-05-23 13:20:28 +00:00
ea87c335d6 Add a roadmap to implement the high-priority items revealed by the code quality audit 2026-05-23 08:10:00 -05:00
7169ff04df Audit code quality and deduplication opportunities 2026-05-23 08:08:24 -05:00
ef1f650bc0 Mark documentation roadmap complete after final validation sweep 2026-05-23 13:04:16 +00:00
0d02cb9fa0 Rewrite integration documentation and verify maintained examples 2026-05-23 13:01:17 +00:00
0299b128cf Rewrite internal documentation for current stage and state contracts 2026-05-23 12:57:59 +00:00
d723384888 Rewrite user and operator documentation for current CLI and config behavior 2026-05-23 12:50:46 +00:00
54228055c8 Audit Stage 1 documentation scope and fix broken references 2026-05-23 12:43:50 +00:00
23ed716450 Added a documentation update roadmap 2026-05-23 07:38:17 -05:00
ab59bab044 Initial documentation cleanup pass 2026-05-23 07:11:17 -05:00
301 changed files with 31145 additions and 10820 deletions

BIN
.DS_Store vendored

Binary file not shown.

View File

@@ -2,8 +2,33 @@ when:
- event: tag
steps:
- name: build-release-assets
validate:
image: golang:1.25
commands:
- go test ./...
- go test -race ./...
- go vet ./...
- go build ./...
- go test ./internal/doccheck
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
cross-build:
image: golang:1.25
depends_on: validate
commands:
- |
set -eu
output_dir="$(mktemp -d)"
trap 'rm -rf "$output_dir"' EXIT
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
goos="${target%/*}"
goarch="${target#*/}"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
done
build-release-assets:
image: golang:1.25
depends_on: [validate, cross-build]
commands:
- |
set -eu
@@ -33,7 +58,7 @@ steps:
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release
publish-release:
image: woodpeckerci/plugin-release
depends_on:
- build-release-assets

8
.woodpecker/shuffle.yml Normal file
View File

@@ -0,0 +1,8 @@
when:
- event: cron
steps:
shuffled-race-tests:
image: golang:1.25
commands:
- go test -race -shuffle=on -count=3 ./...

47
.woodpecker/verify.yml Normal file
View File

@@ -0,0 +1,47 @@
when:
- event: [push, pull_request]
steps:
tests:
image: golang:1.25
commands:
- go test ./...
race-tests:
image: golang:1.25
depends_on: tests
commands:
- go test -race ./...
static-analysis:
image: golang:1.25
depends_on: tests
commands:
- go vet ./...
build:
image: golang:1.25
depends_on: tests
commands:
- go build ./...
documentation-and-examples:
image: golang:1.25
depends_on: tests
commands:
- go test ./internal/doccheck
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
cross-build:
image: golang:1.25
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
commands:
- |
set -eu
output_dir="$(mktemp -d)"
trap 'rm -rf "$output_dir"' EXIT
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
goos="${target%/*}"
goarch="${target#*/}"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
done

View File

@@ -1,22 +1,42 @@
# narratio
Narratio is a Go orchestration application that turns D&D session audio into polished transcripts and generated session artifacts.
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
polished transcripts, validated Notarius extraction lanes, and generated
artifacts.
It coordinates transcription, merge/polish/normalize/trim processing, artifact generation, publish-stage uploads, and resumable run state in one operator workflow.
It runs a deterministic workflow with manifest-driven continuation, remote
publish, and restore support.
```bash
```sh
narratio run 2026-04-04
```
This command requires discoverable `pipeline.yml` and `session.yml` files (or explicit `--config` and `--session` flags).
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete
`session.yml` files or their explicit command-line alternatives.
## Documentation
- [Configuration](docs/config.md)
- [CLI Reference](docs/cli.md)
- [Operations and Recovery](docs/operations.md)
- [Troubleshooting](docs/troubleshooting.md)
- [Development Guide](docs/development.md)
- [Architecture Principles](docs/architecture.md)
- [Internal Component Contracts](docs/internal/README.md)
- [Config Examples](examples/)
- [CLI reference](docs/cli.md) — commands, arguments, flags, and invocation
behavior.
- [Configuration](docs/config.md) — discovery, fields, defaults, and
validation.
- [Operations](docs/operations.md) — runtime workflow, state, publishing,
recovery, and cleanup.
- [Troubleshooting](docs/troubleshooting.md) — symptom-driven diagnosis and
safe remedies.
- [Integration contracts](docs/integrations/) — external tools, formats, and
compatibility expectations.
- [Maintained examples](examples/README.md) — complete copyable configuration
and input files.
## Maintainer Documentation
- [Development guide](docs/development.md) — first-read orientation and
task-specific reading routes.
- [Internal overview](docs/internal/overview.md) — implemented component map.
- [Architecture](docs/policy/architecture.md) — normative boundaries and
invariants.
- [Documentation policy](docs/policy/documentation.md) — canonical ownership
and maintenance rules.
- [Testing policy](docs/policy/testing.md) — test value, boundaries, and
sufficiency.

View File

@@ -1,202 +0,0 @@
# Narratio Architecture
## Purpose
`narratio` is a Go orchestration application for processing D&D session audio into polished transcripts and generated session artifacts.
This document defines the development principles for the project. It is inward-facing: its audience is developers and LLM coding agents. It should guide future changes, not serve as a complete implementation reference.
Implemented component details belong under `docs/internal/`.
## Project Shape
Narratio is a modular, stage-driven orchestrator.
It coordinates specialized downstream systems rather than reimplementing their domains:
- WhisperX handles transcription.
- Seriatim handles deterministic transcript merge/normalization/trim behavior.
- Audita handles transcript correction and polishing.
- Scriptorium handles prompt execution and generated artifacts.
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics.
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
## Core Principles
### Modular and composable
Code should be organized around clear responsibilities. Stages, adapters, config loading, manifest persistence, path construction, and storage behavior should remain separable and independently testable.
### Hexagonal boundaries
External systems should be isolated behind narrow adapters. Stage logic should depend on Narratio-level interfaces and data structures, not on external SDK types, subprocess argument construction, or transport-specific details.
### Standard library preference
Prefer the Go standard library. Add dependencies only when they provide substantial value, are necessary for an external integration, or are a widely used de facto standard.
Accepted examples include a YAML library for configuration and the AWS SDK for S3-compatible storage.
### Explicit orchestration
The pipeline should remain stage-driven and explicit. New behavior should be added through clear stage, adapter, config, or manifest contracts rather than implicit side effects or generic workflow abstraction.
## Stage Design
Each stage should have a clear scope of responsibility.
A stage should define:
- its purpose;
- required input state;
- produced output state;
- config fields it consumes;
- external adapters it uses;
- manifest refs it reads or writes;
- skip, force, and resume behavior;
- failure behavior;
- tests that protect its contract.
Stages should avoid reaching across boundaries. If shared behavior is needed, prefer a helper or service with a narrow interface over duplicating ad hoc logic between stages.
## Transactionality and Resume
A stage should behave transactionally.
A stage is complete only when its outputs have been written, validated, and recorded in the manifest. If a stage fails, Narratio should preserve enough local state for inspection, recovery, and resume.
A failed or incomplete run must not be treated as successful. Later stages should depend on manifest-recorded success, not merely on incidental files existing on disk.
## Manifest Model
The manifest is the durable local ledger for a run.
It should record:
- run identity;
- stage status;
- input and output refs;
- logs and generated config refs;
- checksums or provenance where useful;
- non-secret adapter and publish metadata.
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
## Adapter Boundaries
Adapters own external integration details.
Expected boundaries:
- WhisperX HTTP details stay in the WhisperX adapter.
- Seriatim CLI construction stays in the Seriatim adapter.
- Audita CLI construction stays in the Audita adapter.
- Scriptorium CLI construction stays in the Scriptorium adapter.
- Object-storage details stay behind the storage adapter interface.
- AWS SDK types stay inside the S3 storage implementation.
Stage code should express intent in Narratio terms and call adapters through narrow contracts.
## Configuration Philosophy
Configuration should be strict, explicit, and operator-friendly.
Principles:
- YAML decoding should reject unknown fields.
- Defaults should be centralized and testable.
- Empty configured values should not silently override meaningful defaults.
- Session templating should remain narrow and deterministic.
- Template support should serve operator convenience, not become a general configuration language.
Narratio should not become a secondary configuration system for downstream tools. Seriatim, Audita, and Scriptorium should own their runtime defaults wherever practical. Narratio should pass required stage-contract paths and explicit operator overrides.
## Path and Storage Discipline
Local and remote paths are part of Narratios application contract.
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
## Publish Invariants
Publish behavior must preserve a clear commit boundary.
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`.
`current/run_id.txt` is the final remote commit marker and must be written last.
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured.
## Security and Privacy
Narratio handles private campaign material.
Rules:
- Do not store raw secrets in pipeline or session YAML.
- Use environment variable names or secret-file references for secret handling.
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
## Diagnostics
Diagnostics should be durable and discoverable, but distinct from canonical outputs.
Logs, reports, generated invocation/config files, and render-debug files support debugging. Transcript tiers and configured artifacts are pipeline products.
Manifest refs should preserve that distinction.
## Determinism
Where practical, Narratio should prefer deterministic behavior:
- stable local path layout;
- stable remote key layout;
- sorted upload order;
- predictable generated config files;
- repeatable command construction;
- tests that do not depend on live external services.
Run IDs and timestamps may be intentionally variable, but surrounding behavior should remain testable.
## Testing Expectations
Core behavior should be testable without live external services.
Tests should cover:
- config loading, defaults, and validation;
- CLI parsing and command construction;
- path helpers;
- manifest transitions;
- stage success, failure, skip, and resume behavior;
- adapter command construction;
- fake storage behavior;
- publish commit ordering;
- example config validity where practical.
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
## Documentation Expectations
Documentation must follow `docs/documentation/policy.md`.
Current behavior belongs in user-facing docs and `docs/internal/`. Future, planned, aspirational, experimental, or unimplemented work belongs only under `docs/roadmap/`.
`docs/architecture.md` should remain concise and principle-focused. It should not duplicate the full config reference, CLI reference, operations guide, or internal stage documentation.
## Non-Goals
Narratio is not:
- a generic DAG or workflow engine;
- a replacement configuration layer for Seriatim, Audita, or Scriptorium;
- a storage backend abstraction beyond the needs of this pipeline;
- a place to embed raw secrets;
- a place for stage logic to depend directly on AWS SDK types or downstream tool internals;
- a prompt-authoring system.

View File

@@ -1,4 +1,4 @@
# CLI
# CLI Reference
## Shortest Useful Command
@@ -6,18 +6,18 @@
narratio run 2026-04-04
```
This runs the full pipeline for the given session ID using default config discovery and campaign selection.
This runs the canonical full pipeline for session `2026-04-04`.
## Command Overview
Top-level commands:
- `run <session_id>`: execute the pipeline.
- `resume <session_id>`: continue from first non-succeeded stage.
- `run-stage <stage> <session_id>`: execute exactly one stage.
- `analyze <session_id>`: force-rerun analyze stage.
- `publish <session_id>`: force-rerun publish stage.
- `clean <session_id>|--all`: remove local workspace/spool state.
- `session <subcommand>`: session-scoped helper commands.
- `run <session_id>`: run full stage order.
- `run-stage <stage> <session_id>`: run one stage.
- `analyze <session_id>`: force-run analyze.
- `publish <session_id>`: force-run publish.
- `clean <session_id>` or `clean --all`: remove local work/spool state.
- `session <subcommand>`: session helper commands.
Session subcommands:
@@ -31,39 +31,65 @@ Session subcommands:
- `session locks add <session_id> <source>`
- `session locks remove <session_id> <source>`
## Common Flags
## Common Config Flags
Most session-aware commands accept:
- `--config <pipeline.yml>`
- `--campaign <id>`
- `--campaign-file <campaign.yml>`
- `--session <session.yml>`
- `--previous-session-id <id>`
- `--session-id <session_id>`
- `--previous-session-id <session_id>`
`--campaign` and `--campaign-file` are mutually exclusive.
Rules:
- `--campaign` and `--campaign-file` are mutually exclusive.
- `--session` is not used by `session init`.
- if both positional `<session_id>` and `--session-id` are provided, values must match.
- `--previous-session-id` is a strict expectation: the selected session file
must contain the same `previous_session_id`.
- `clean --all` cannot be combined with campaign/session selectors.
- notification delivery is currently limited to the configured `noop` mode; see
the [configuration reference](./config.md#notifications).
## Session ID Input Rules
Session-aware commands accept one of these forms:
- positional session ID: `... <session_id>`
- compatibility flag: `... --session-id <session_id>`
When both are present, command parsing requires an exact match.
Commands with additional positionals keep their command-specific order:
- `run-stage <stage> <session_id>` or `run-stage <stage> --session-id <session_id>`
- `session locks add <session_id> <source>` or `session locks add --session-id <session_id> <source>`
- `session locks remove <session_id> <source>` or `session locks remove --session-id <session_id> <source>`
## Command Reference
### `run`
```bash
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
```
Runs stages in canonical order and writes manifest state.
Behavior:
### `resume`
```bash
narratio resume <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
```
Starts at the first non-succeeded stage from the session manifest.
- evaluates full stage order;
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
configuration records an explicit `notarius_disabled` self-skip;
- skips already-succeeded stages unless `--force` is set or a stage-specific
resume check finds its durable result obsolete;
- continues interrupted or partially completed sessions by running non-succeeded stages;
- writes session and run manifests.
### `run-stage`
```bash
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common flags]
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
```
Valid stage names:
@@ -74,127 +100,170 @@ Valid stage names:
- `polish`
- `normalize`
- `trim`
- `extract`
- `render`
- `analyze`
- `publish`
- `notify`
`--artifacts` is accepted only for `analyze` and `publish`.
Rules:
- `--artifacts` is accepted only for `analyze` and `publish` stage targets.
### `analyze`
```bash
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common flags]
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common config flags]
```
Equivalent to `narratio run-stage analyze <session_id> --force`.
Equivalent to:
```bash
narratio run-stage analyze <session_id> --force [...common config flags]
```
### `publish`
```bash
narratio publish <session_id> [--artifacts <name[,name...]>] [...common flags]
narratio publish <session_id> [--artifacts <name[,name...]>] [...common config flags]
```
Equivalent to `narratio run-stage publish <session_id> --force`.
Equivalent to:
```bash
narratio run-stage publish <session_id> --force [...common config flags]
```
### `clean`
```bash
narratio clean <session_id> [--dry-run] [--clear-cache] [...common flags]
narratio clean <session_id> [--dry-run] [--clear-cache] [...common config flags]
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
```
- session mode deletes `{workspace.root}/work/{campaign}/{session_id}` and `{spool.root}/{campaign}/{session_id}`.
- `--all` deletes all session work and spool children.
- cache is preserved unless `--clear-cache` is passed.
Behavior:
- session mode removes the selected session's local work and spool state;
- `--all` removes all local session work and spool state;
- cache remains unless `--clear-cache` is provided.
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
post-publish cleanup behavior.
### `session plan`
```bash
narratio session plan <session_id> [--force] [...common flags]
narratio session plan <session_id> [--force] [...common config flags]
```
Validates config and session inputs, prepares workdir layout, and prints stage run/skip decisions.
Validates config, prepares local workdir layout, and prints run/skip decisions for each stage.
### `session validate`
```bash
narratio session validate <session_id> [...common flags]
narratio session validate <session_id> [...common config flags]
```
Read-only preflight checks for config, inputs, audio availability, previous-session requirements, publish outputs, and effective locks.
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
### `session status`
```bash
narratio session status <session_id> [...common flags]
narratio session status <session_id> [...common config flags]
```
Shows local manifest state, remote current state (when storage is configured), published-output availability, and effective locks.
Prints local manifest state and, when storage is available, status for the
pointer-selected remote commit and its declared published outputs.
### `session init`
```bash
narratio session init <session_id> --output ./session.yml
narratio session init <session_id> --remote
narratio session init <session_id> --remote --force
narratio session init <session_id> --output ./session.yml [options]
narratio session init <session_id> --remote [options]
```
Flags:
Required target selection:
- `--output <path>` or `--remote` (exactly one is required)
- exactly one of:
- `--output <path>`
- `--remote`
Options:
- `--config <pipeline.yml>`
- `--campaign <id>` or `--campaign-file <campaign.yml>`
- `--previous-session-id <id>`
- `--date <date>`
- `--title <title>`
- `--date <YYYY-MM-DD>`
- `--title <text>`
- `--audio-dir <path>`
- `--audio-s3-prefix <prefix>`
- `--force`
- common config/campaign flags
Rules:
- `--audio-dir` and `--audio-s3-prefix` are mutually exclusive.
- if campaign `session_template_file` is configured, `session init` renders it.
- generated session YAML must be concrete (no unresolved `{{ ... }}` placeholders).
### `session restore`
```bash
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common flags]
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common config flags]
```
Restores durable local session files from committed remote current state.
Behavior:
Default restore scope:
- discovers committed remote current state;
- plans local restores;
- writes an execution report;
- blocks unresolved conflicts. `--force` permits replacement only of eligible
regular files.
- `manifest.json`
- `transcripts/**`
- `artifacts/**`
- `previous/**` when required by configured previous-session artifact inputs
`audio/**` is restored only when `--include-audio` is set.
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
default restore scope, report location, and conflict-handling workflow.
### `session artifacts`
```bash
narratio session artifacts <session_id> [--remote] [...common flags]
narratio session artifacts <session_id> [--remote] [...common config flags]
```
Lists built-in sources, configured artifact sources, previous-session sources, publish output rules, and lock status. With `--remote`, includes remote published-state markers.
Lists effective built-in, configured Scriptorium, and configured extraction
sources; reports planned, available, unavailable, and published state without
reading payload bodies; and includes publish rules, lock state, and optional
remote published-state availability.
### `session locks`
```bash
narratio session locks <session_id> [...common flags]
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common flags]
narratio session locks remove <session_id> <source> [...common flags]
narratio session locks <session_id> [...common config flags]
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common config flags]
narratio session locks remove <session_id> <source> [...common config flags]
```
- list mode prints effective locks from static `pipeline.publish.locks` and remote `{session_prefix}/locks.yml`.
- add/remove mutate only the remote lock store.
- static pipeline locks cannot be removed by lock commands.
Behavior:
## `--artifacts` Rules
- list mode reports the effective merge of static and remote locks;
- add/remove mutate only remote locks;
- static locks from pipeline config cannot be removed by CLI commands.
- accepted on `run`, `resume`, `run-stage`, `analyze`, and `publish`.
- on `run-stage`, only valid for `analyze` and `publish`.
- filters configured analyze artifact execution.
- filters configured `pipeline.publish.outputs` entries for `narratio.artifact.<key>` sources.
- does not suppress built-in transcript/bounds publish outputs.
- does not imply `--force` for `run`, `resume`, or `run-stage`.
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
and precedence.
## `--artifacts` Selection Rules
- accepted on `run`, `run-stage`, `analyze`, and `publish`;
- names must exist in `pipeline.scriptorium.artifacts`;
- empty entries are invalid;
- repeated names are deduplicated.
Effects:
- filters analyze execution to selected configured artifacts;
- filters publish rules that source `narratio.artifact.<name>`;
- does not filter built-in transcript/bounds or explicitly configured
`narratio.extraction.<name>` publish sources; and
- does not select or filter Notarius lanes.
## Common Workflows
@@ -204,16 +273,16 @@ Run full pipeline:
narratio run 2026-04-04
```
Run only selected analyze artifacts:
Dry-run restore plan:
```bash
narratio run 2026-04-04 --artifacts session_recap,player_handout
narratio session restore 2026-04-04 --dry-run
```
Force analyze only:
Generate a concrete session file from template/default structure:
```bash
narratio analyze 2026-04-04 --artifacts player_handout
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
```
Force publish only:
@@ -222,9 +291,17 @@ Force publish only:
narratio publish 2026-04-04
```
Restore preview then apply:
## Output And Exit Behavior
```bash
narratio session restore 2026-04-04 --dry-run
narratio session restore 2026-04-04
```
- Successful commands write their result or summary to standard output and
exit with status `0`.
- Command failures and invalid invocations write an error to standard error and
exit with status `1`.
- An unknown top-level command also prints the top-level usage summary to
standard error.
- `session restore --help` prints its command-specific usage and exits with
status `0`.
Output is intended for operator inspection. Narratio does not currently offer
a machine-readable CLI output mode; durable machine-readable state is recorded
in manifests and reports described in [Operations](./operations.md).

View File

@@ -1,56 +1,72 @@
# Configuration
# Configuration Reference
## Overview
Narratio loads three YAML files:
## Purpose
- `pipeline.yml`: pipeline-level runtime settings.
- `campaign.yml`: stable campaign identity and campaign-level input defaults.
- `session.yml`: per-session metadata and input selection.
Narratio resolves three YAML documents:
Commands that load and validate all three files include:
- `pipeline.yml`: pipeline/runtime settings
- `campaign.yml`: campaign identity and stable input defaults
- `session.yml`: session identity, metadata, and audio source selection
- `narratio run`
- `narratio resume`
- `narratio run-stage`
- `narratio analyze`
- `narratio publish`
- `narratio session plan`
- `narratio session status`
- `narratio session validate`
- `narratio session restore`
- `narratio session artifacts`
- `narratio session locks`
- `narratio clean <session_id>`
## Discovery and Selection
Validation behavior:
### `pipeline.yml`
- strict YAML decode is enabled (`KnownFields(true)`); unknown fields fail.
- loaded `session.yml` files must be concrete YAML (no `{{ ... }}` placeholders).
- defaults are applied for optional pipeline fields.
- campaign/session identity mismatches fail load.
## File Discovery
Pipeline discovery order when `--config` is omitted:
When `--config` is omitted, search order is:
1. `/usr/local/etc/narratio/pipeline.yml`
2. `/etc/narratio/pipeline.yml`
Session discovery order when `--session` is omitted:
### `campaign.yml`
Selection rules:
- if `--campaign-file` is set, use that path;
- else if `--campaign <id>` is set, use `{pipeline.campaigns.root}/{id}/campaign.yml`;
- else use `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`.
### `session.yml`
When `--session` is omitted, local search order is:
1. `/usr/local/etc/narratio/session.yml`
2. `/etc/narratio/session.yml`
Campaign discovery when `--campaign-file` is omitted:
If local session discovery fails and a `session_id` is known, Narratio attempts remote session loading from:
- if `--campaign <id>` is set: `{pipeline.campaigns.root}/{id}/campaign.yml`
- otherwise: `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`
Remote `session.yml` fallback:
- if local session discovery fails and storage is configured, Narratio can load:
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
## Minimal Working Config
using configured object storage.
The downloaded remote session file is command-scoped: Narratio removes it after
the command finishes and records only the remote object provenance alongside
the durable copied session input.
### Identity segments
Campaign IDs (`campaign_id` and `default_campaign_id`), session IDs, previous
session IDs, and Narratio run IDs are opaque portable segments. They must use
only ASCII letters, digits, `.`, `_`, and `-`; empty values, `.`/`..`, path
separators, drive forms, whitespace, control characters, and non-ASCII text are
rejected. Narratio does not trim or rewrite these values. Existing manifests or
remote state with an unsafe legacy identity must be migrated before use.
## Validation and Merge Rules
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
unknown fields or trailing documents fail load.
- Configured timeout and retry-delay durations must be positive. An omitted
artifact timeout continues to inherit its configured Scriptorium timeout.
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
- Pipeline defaults are applied before validation.
- Campaign and session identities must agree.
- Stable files (`speakers_file`, `autocorrect_file`, `glossary_file`, `players_file`, `party_file`) resolve from session overrides when provided, otherwise from campaign defaults.
- Exactly one audio mode must be configured in session input:
- local (`audio_dir` or `audio_files`), or
- S3 (`audio_s3.prefix`).
## Minimal Working Configuration
`pipeline.yml`
```yaml
@@ -58,7 +74,7 @@ campaigns:
root: /usr/local/share/narratio/campaigns
default_campaign_id: sample-campaign
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
transcribe_url: https://transcription.example.com/transcribe
```
`campaign.yml`
@@ -69,9 +85,11 @@ inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
players_file: ./players.yml
party_file: ./party.yml
```
`session.yml`
`session.yml` (local audio)
```yaml
session_id: 2026-05-03
@@ -79,8 +97,22 @@ inputs:
audio_dir: ./audio
```
## Publish Config
Top-level publish settings live at `pipeline.publish`.
## Secrets Handling
- Do not place raw secrets in YAML.
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
environment-variable filename supplies one value; trailing CR/LF is removed.
- An existing process environment value takes precedence over a credential file.
- Credential directories and files must not be symlinks and must be regular,
bounded files (at most 8 KiB per value). On POSIX, provision the directory
with no group/other access (normally `0700`) and files with no group/other
access (normally `0600`).
- Commands that need storage/auth load filesystem secrets before constructing adapters.
## Publish Configuration Summary
Publish rules live under `pipeline.publish`.
```yaml
publish:
@@ -90,39 +122,43 @@ publish:
- source: narratio.transcript.final_trimmed
dest: transcripts/final.trimmed.json
required: true
- source: narratio.transcript.final_markdown
dest: transcripts/final.md
required: true
- source: narratio.transcript.final_trimmed_markdown
dest: transcripts/final.trimmed.md
required: true
- source: narratio.artifact.session_recap
dest: artifacts/session_recap.md
required: true
locks:
- source: narratio.artifact.session_recap
reason: Final recap was manually edited.
reason: manual post-publish edits
```
Rules:
- `outputs[].source` is required.
- `outputs[].dest` is optional; when omitted, Narratio derives destination from the source.
- `outputs[].dest` may be omitted when derivable from source.
- extraction sources require an explicit `outputs[].dest` and publish only when
a rule names that source; the Notarius index and complete bundle are not
publish sources.
- `outputs[].required` defaults to `true`.
- static `publish.locks` and remote `{session_prefix}/locks.yml` are merged; static locks win on duplicates.
- locks prevent overwrite of top-level published destinations.
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
Supported publish source families:
- built-ins: `narratio.transcript.base`, `narratio.transcript.polished`, `narratio.transcript.final`, `narratio.transcript.final_trimmed`, `narratio.bounds.session`
- configured artifacts: `narratio.artifact.<artifact_key>`
## Full Reference
## Full Schema
### Pipeline
| Path | Type | Required | Default |
| Field | Type | Required | Default / Rule |
| --- | --- | --- | --- |
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
| `pipeline.secrets.env_dir` | string | No | empty |
| `pipeline.storage.backend` | string | No | empty |
| `pipeline.storage.s3.bucket` | string | Conditional | empty |
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
| `pipeline.storage.s3.region` | string | No | empty |
| `pipeline.storage.s3.endpoint` | string | No | empty |
@@ -135,21 +171,21 @@ Supported publish source families:
| `pipeline.cache.s3_audio` | bool | No | `true` |
| `pipeline.publish.enabled` | bool | No | `true` |
| `pipeline.publish.upload_run` | bool | No | `true` |
| `pipeline.publish.outputs[]` | list | No | one final-trimmed output rule |
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | none |
| `pipeline.publish.outputs[].dest` | string | No | derived from source |
| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed JSON plus final and final-trimmed Markdown outputs |
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
| `pipeline.publish.outputs[].required` | bool | No | `true` |
| `pipeline.publish.locks[]` | list | No | empty |
| `pipeline.publish.locks[].source` | string | Yes (per lock) | none |
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
| `pipeline.publish.locks[].reason` | string | No | empty |
| `pipeline.whisperx.transcribe_url` | string | Yes | none |
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
| `pipeline.whisperx.language` | string | No | `en` |
| `pipeline.whisperx.timeout` | duration string | No | `30m` |
| `pipeline.whisperx.timeout` | duration | No | `30m` |
| `pipeline.whisperx.retries` | int | No | `3` |
| `pipeline.whisperx.retry_delay` | duration string | No | `2s` |
| `pipeline.whisperx.retry_delay` | duration | No | `2s` |
| `pipeline.whisperx.concurrency` | int | No | `2` |
| `pipeline.seriatim.binary` | string | No | `seriatim` |
| `pipeline.seriatim.timeout` | duration string | No | `10m` |
| `pipeline.seriatim.timeout` | duration | No | `10m` |
| `pipeline.seriatim.output_schema` | string | No | `seriatim-intermediate` |
| `pipeline.seriatim.coalesce_gap` | float | No | `3.0` |
| `pipeline.seriatim.report` | bool | No | `true` |
@@ -158,7 +194,7 @@ Supported publish source families:
| `pipeline.seriatim.env.backchannel_max_duration` | float | No | unset |
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
| `pipeline.audita.binary` | string | No | `audita` |
| `pipeline.audita.timeout` | duration string | No | `3h` |
| `pipeline.audita.timeout` | duration | No | `3h` |
| `pipeline.audita.llm_api_key_env` | string | No | empty |
| `pipeline.audita.modules[]` | list[string] | No | empty |
| `pipeline.audita.base_url` | string | No | empty |
@@ -175,53 +211,152 @@ Supported publish source families:
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
| `pipeline.normalize.report` | bool | No | `true` |
| `pipeline.trim.enabled` | bool | No | `false` |
| `pipeline.trim.output_path` | string | Conditional | none |
| `pipeline.trim.bounds.prompt_id` | string | Conditional | none |
| `pipeline.trim.enabled` | bool | No | `true` |
| `pipeline.trim.output_path` | string | No | `transcripts/final.trimmed.json` |
| `pipeline.trim.bounds.prompt_id` | string | No | `dnd.session_bounds` |
| `pipeline.trim.bounds.profile_id` | string | No | empty |
| `pipeline.trim.bounds.transcript_input_name` | string | Conditional | none |
| `pipeline.trim.bounds.output_path` | string | Conditional | none |
| `pipeline.trim.bounds.timeout` | duration string | No | `10m` |
| `pipeline.trim.bounds.transcript_input_name` | string | No | `transcript` |
| `pipeline.trim.bounds.output_path` | string | No | `artifacts/session_bounds.json` |
| `pipeline.trim.bounds.timeout` | duration | No | `10m` |
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
| `pipeline.trim.bounds.render_output_path` | string | Conditional | none |
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
| `pipeline.trim.seriatim.report` | bool | No | `false` |
| `pipeline.notarius.enabled` | bool | No | `false` |
| `pipeline.notarius.binary` | string | No | `notarius` |
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
| `pipeline.render.enabled` | bool | No | `true` |
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
| `pipeline.render.include_timestamps` | bool | No | `true` |
| `pipeline.render.include_segment_ids` | bool | No | `true` |
| `pipeline.render.include_metadata` | bool | No | `false` |
| `pipeline.scriptorium.binary` | string | No | `scriptorium` |
| `pipeline.scriptorium.config_path` | string | No | empty |
| `pipeline.scriptorium.timeout` | duration string | No | `10m` |
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
| `pipeline.scriptorium.artifacts` | map | No | empty |
| `pipeline.notification.backend` | string | No | empty |
| `pipeline.notification.recipient` | string | No | empty |
| `pipeline.notification.timeout` | duration string | No | `30s` |
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
### Notarius Output Entries
For each `pipeline.notarius.outputs.<name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `lane_id` | string | Yes | unique Notarius lane ID |
| `media_type` | string | Yes | exact accepted descriptor media type |
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
| `schema_version` | string | Yes | exact accepted descriptor schema version |
| `module_key` | string | No | exact accepted module key when set |
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
required from a successful Notarius result; a missing, rejected, duplicate, or
contract-incompatible lane fails extraction. See the
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
for compatibility ownership.
### Scriptorium Artifact Entries
For each `pipeline.scriptorium.artifacts.<name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `enabled` | bool | No | `false` if omitted |
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; enabled graph must be acyclic |
| `render_debug` | bool | No | per-artifact override |
| `prompt_id` | string | Conditional | required when artifact is enabled |
| `profile_id` | string | No | empty |
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
| `timeout` | duration | No | artifact override |
| `inputs` | map | No | input key names must be non-empty |
| `vars` | map | No | values must be string or bool; `session_id` is reserved and overwritten by Narratio |
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
Without `--artifacts`, analyze executes enabled configured artifacts. With an
explicit `--artifacts` list, the exact named configured artifacts are the
one-invocation execution set even if their `enabled` values are false; the list
does not automatically include dependencies. Named artifacts must therefore be
configured with valid executable fields, and their configured dependencies must
already be available to analyze. This override affects analyze planning only;
publish uses the list only to filter configured
`narratio.artifact.<name>` output rules.
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
| `required` | bool | No | optional input requirement |
`artifact` and `path` are obsolete and rejected by strict configuration
loading. Use the canonical `source` identifier to select the input; Narratio
does not provide adapter-specific input passthrough fields.
### Notifications
Narratio currently supports only `notification.mode: noop`, which is also the
default when the section is omitted. The notify stage performs no delivery in
this mode. Backend, recipient, timeout, and other provider settings are
rejected by strict configuration loading until Narratio has a provider
integration.
### Campaign
| Path | Type | Required |
| --- | --- | --- |
| `campaign_id` | string | Yes |
| `session_template_file` | string | No |
| `inputs.speakers_file` | string | Yes |
| `inputs.autocorrect_file` | string | Yes |
| `inputs.glossary_file` | string | Yes |
| Field | Type | Required | Notes |
| --- | --- | --- | --- |
| `campaign_id` | string | Yes | canonical opaque campaign identity |
| `session_template_file` | string | No | used by `session init` when set |
| `inputs.speakers_file` | string | Yes | stable input default |
| `inputs.autocorrect_file` | string | Yes | stable input default |
| `inputs.glossary_file` | string | Yes | stable input default |
| `inputs.players_file` | string | Yes | stable input default |
| `inputs.party_file` | string | Yes | stable input default |
### Session
| Path | Type | Required |
| --- | --- | --- |
| `session_id` | string | Yes |
| `previous_session_id` | string | No |
| `campaign` | string | No |
| `date` | string | No |
| `title` | string | No |
| `inputs.speakers_file` | string | No |
| `inputs.autocorrect_file` | string | No |
| `inputs.glossary_file` | string | No |
| `inputs.audio_dir` | string | Conditional |
| `inputs.audio_files[]` | list[string] | Conditional |
| `inputs.audio_s3.prefix` | string | Conditional |
| Field | Type | Required in session file | Notes |
| --- | --- | --- | --- |
| `session_id` | string | Yes | opaque identity; must match CLI session target when provided |
| `previous_session_id` | string | No | opaque identity; must not equal `session_id` |
| `campaign` | string | No | opaque identity; filled from `campaign_id` during resolve if omitted |
| `date` | string | No | metadata |
| `title` | string | No | metadata |
| `inputs.speakers_file` | string | No | overrides campaign stable input |
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
| `inputs.glossary_file` | string | No | overrides campaign stable input |
| `inputs.players_file` | string | No | overrides campaign stable input |
| `inputs.party_file` | string | No | overrides campaign stable input |
| `inputs.audio_dir` | string | Conditional | local audio mode |
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
Audio rules:
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
- `audio_s3` requires `pipeline.storage.backend: s3` and a configured S3 bucket.
### Storage backend selection
`local` is the default and disables remote object-store operations. Configure
`s3` explicitly before supplying `storage.s3`; a populated S3 block does not
select a backend on its own. Unknown backend names and an S3 block paired with
`local` are rejected during configuration validation.
### Previous-session expectation
`previous_session_id` is optional in a session file. When a command supplies
`--previous-session-id`, however, the session file must contain the same value;
an omitted or different value is rejected before the command performs work.
## Maintained Examples
- `examples/pipeline.minimal.yml`
- `examples/pipeline.production.yml`
- `examples/pipeline.full.annotated.yml`
- `examples/campaigns/sample-campaign/campaign.yml`
- `examples/session.local-audio.yml`
- `examples/session.s3-audio.yml`
See the [maintained examples index](../examples/README.md) for complete pipeline,
campaign, session, template, and input fixtures. Keep complete copyable files
there rather than duplicating them in this reference.

View File

@@ -1,94 +1,58 @@
# Development Guide
# Development
## Purpose
Canonical contributor workflow and engineering conventions for implemented Narratio behavior.
This is the first-read landing page for people and LLM coding agents working on
Narratio. It provides a concise repository orientation and routes each kind of
change to its canonical documentation.
## Repository layout
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
polished transcripts and generated artifacts. Start with the
[README](../README.md) for product context,
[Architecture](policy/architecture.md) for normative system boundaries, and the
[Internal Overview](internal/overview.md) for implemented component ownership.
- `cmd/narratio/`: CLI entrypoint.
- `internal/app/`: command handlers, plan/run/resume orchestration, cleanup gates, secrets loading.
- `internal/config/`: strict YAML loading, defaults, and validation.
- `internal/stage/`: stage implementations and stage registry/order.
- `internal/adapters/`: external boundary adapters (WhisperX, Seriatim, Audita, Scriptorium, storage, notify).
- `internal/manifest/`: session/run manifest types and persistence.
- `internal/artifacts/`: canonical local/remote path helpers and local artifact store.
- `docs/`: canonical documentation set.
- `examples/`: maintained config examples used by tests.
## What To Read
## Build and test commands
| When working on | Read | Why |
| --- | --- | --- |
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internal documents. |
| Application shape, boundaries, dependency direction, runtime invariants, safety properties, or dependencies | [Architecture](policy/architecture.md) | It defines the intended system shape, ownership, and non-goals. |
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical owners, audiences, current-behavior rules, and maintenance requirements. |
| Adding, changing, reviewing, rewriting, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and test lifecycle decisions. |
| CLI composition or command behavior | [Internal Overview](internal/overview.md) and [CLI Reference](cli.md) | The overview routes to command ownership; the reference owns public syntax and invocation behavior. |
| Configuration loading, resolution, or user-visible configuration | [Internal Overview](internal/overview.md) and [Configuration](config.md) | The overview routes to implementation ownership; the reference owns fields, defaults, discovery, and validation. |
| Session workflow, status, restore, cleanup, or object storage | [Restore Internals](internal/command-restore.md), [Workspace Internals](internal/workspace.md), [Storage Internals](internal/storage.md), [Operations](operations.md), and [Troubleshooting](troubleshooting.md) | These separate implementation mechanics, operator procedures, and symptom-driven recovery. |
| Pipeline sequencing or the behavior of a stage | [Internal Overview](internal/overview.md) and its focused stage documents | The overview owns the implemented stage inventory and routes to each stage contract. |
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
- Run focused CLI behavior checks:
For an existing subsystem, also inspect its focused tests and package-level
contracts before changing behavior.
```bash
go test ./internal/app -run TestExecute -v
```
## Validation
- Run config example load/validate checks:
Use focused package tests while iterating. Every pull request and push runs the
following repository-wide checks before it can be accepted:
```bash
go test ./internal/config -run TestExamplesLoadAndValidate -v
```
- Run full test suite:
```bash
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go test ./internal/doccheck
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
```
## Coding conventions
The documentation check verifies local Markdown links and the dependency graph
of the Woodpecker workflows. The configuration check loads every maintained
pipeline and session example. Release automation repeats these checks and
cross-compiles the CLI before it builds release assets; publishing depends on
that validation path, so a failure cannot publish a release.
- Keep orchestration explicit and stage-driven; do not introduce generic workflow/DAG abstractions.
- Keep external-system details inside adapter packages; stages should consume Narratio-level contracts only.
- Use centralized path helpers from `internal/artifacts` rather than ad hoc path concatenation.
- Preserve manifest-driven state transitions (`running`, `succeeded`, `failed`, `skipped`, `stale`) as the source of run progress.
- Keep user/operator docs implementation-accurate; planned work belongs only under `docs/roadmap/`.
For design principles and invariants, see [docs/architecture.md](./architecture.md). For stage/adapter contracts, see [docs/internal/README.md](./internal/README.md).
## Dependency policy
- Prefer Go standard library where practical.
- Add third-party dependencies only when they provide clear value for required behavior.
- Keep dependency additions narrow to the boundary package that needs them.
## Change playbooks
### Add config fields
1. Add fields to config structs in `internal/config`.
2. Set defaults in `internal/config/defaults.go` when appropriate.
3. Add validation rules in `internal/config/validate.go`.
4. Add or update load/validate tests in `internal/config/*_test.go`.
5. Update canonical config docs and examples:
- [docs/config.md](./config.md)
- relevant files under `examples/`
### Add CLI flags or commands
1. Update command parsing and behavior in `internal/app`.
2. Add or update command tests (`TestExecute` and command-specific tests).
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
### Add or modify stages/adapters
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
2. Keep external transport/subprocess details in `internal/adapters`.
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
4. Add/update stage and adapter tests.
5. Update internal component contracts in `docs/internal/`.
### Update examples
1. Keep canonical examples only in `examples/`.
2. Ensure examples load and validate through runtime config paths.
3. Update `internal/config/load_validate_test.go` as needed.
4. Update links in `docs/config.md` if example filenames change.
### Update docs and roadmap
1. Keep implemented behavior in canonical docs (`README`, `docs/*.md`, `docs/internal/`).
2. Keep planned/unimplemented behavior only in `docs/roadmap/`.
3. After completing roadmap items, remove or mark them complete in `docs/roadmap/documentation.md`.
4. Run a link/path sweep before finalizing changes.
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
job to expose ordering and repeatability defects. Current runners cross-compile
for macOS and Windows, but do not provide native macOS or Windows execution.
Those cross-builds establish compilation only, not platform-equivalent runtime
evidence. Add native checks only when official runner labels and successful
native-run evidence are available.

View File

@@ -1,356 +0,0 @@
# Go Project Documentation Policy
## Purpose
Project documentation must help four audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
## Core Rules
### 1. Keep docs concise
Each document should cover a defined scope and only the essentials for that scope.
Avoid:
- long background explanations;
- repeated reference material;
- implementation detail in user-facing docs;
- aspirational language outside roadmap docs;
- verbose examples where one minimal example is clearer.
### 2. Document only implemented behavior outside roadmap files
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
- `docs/roadmap/`
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
### 3. Use canonical homes
Each type of information should have one canonical location.
Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/architecture.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- implemented internals: `docs/internal/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/development.md`
- copyable examples: `examples/`
Other files should summarize briefly and link to the canonical source.
### 4. Keep examples real
Examples should be valid, maintained, and free of secrets.
Where practical:
- example configs should load successfully;
- example commands should match real CLI syntax;
- important examples should be covered by tests.
## Documentation Profiles
All projects require:
- `README.md`
- `docs/architecture.md`
Additional docs depend on the project.
### Small library
Recommended:
- `docs/development.md`, if contributor conventions are non-obvious
### Simple CLI
Required:
- `docs/cli.md`
Recommended:
- `docs/development.md`
### Config-driven CLI
Required:
- `docs/cli.md`
- `docs/config.md`
Recommended:
- `examples/`
- `docs/development.md`
### Stateful or operator-facing application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
Recommended:
- `docs/troubleshooting.md`
- `examples/`
- `docs/development.md`
### Modular, staged, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/development.md`
Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
## Required Documents
### README.md
**Audience:** users, administrators, operators
The README is the outward-facing project orientation page.
It should include, in order:
1. concise description;
2. elevator pitch;
3. shortest useful command or usage example;
4. links to targeted docs.
The README should be short. It is not a manual.
The “shortest useful command” means the simplest command that performs the projects core use case. (It does not mean `app --help`.)
### docs/architecture.md
**Audience:** developers, LLM coding agents
`docs/architecture.md` is required for every project.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
It should include:
- project shape;
- core design principles;
- package and boundary philosophy;
- state/persistence philosophy, if applicable;
- external integration philosophy, if applicable;
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/architecture.md` describes development principles.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.

View File

@@ -1,15 +1,35 @@
# Integration Documentation Index
# Integrations Index
## Audience
Developers and LLM coding agents changing Narratio's external integration contracts.
Operators, developers, and coding agents who need to understand Narratio's
externally observable integration boundaries.
## Scope
Implemented-only reference notes for the external systems Narratio currently integrates with.
## Integration Docs
- `audita.md`: Audita adapter invocation and validation contract.
- `seriatim.md`: Seriatim normalize/merge/trim adapter contract.
- `scriptorium.md`: Scriptorium run/render adapter contract.
`docs/integrations/` is the canonical reference for protocols, invocation and
data contracts, logical outputs, and compatibility behavior at external tool
boundaries.
## Canonical Owner
`docs/integrations/` is the canonical home for external integration reference notes per `docs/documentation/policy.md`.
These documents describe what Narratio sends or invokes, what it accepts in
return, and how failures are surfaced. Internal composition and stage mechanics
belong in [the adapter implementation guide](../internal/adapters.md) and the
focused stage documents.
## Integration Contracts
- [Audita](./audita.md): transcript polishing (`audita process`).
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
discovery (`notarius run`).
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
(`scriptorium run|render`).
- [WhisperX](./whisperx.md): speaker-audio transcription over HTTP.
## Related Canonical Docs
- [Configuration](../config.md): operator-facing configuration reference.
- [Adapter implementation](../internal/adapters.md): shared adapter boundary and
runner wiring.
- [Internal documentation](../internal/overview.md): stage-specific integration
usage and component ownership.

View File

@@ -1,66 +1,67 @@
# Integration: audita
# Integration: Audita
## Purpose
Define Narratio's adapter contract for transcript polishing via Audita CLI subprocess execution.
Define the Audita adapter contract used by the `polish` stage.
## Inputs and Outputs
Inputs (`audita.PolishRequest`):
- base transcript path
- glossary path
- output polished transcript path
- optional report path (required when report enabled)
- work dir
- generated config path
- stdout/stderr log paths
- optional module/model/base URL and concurrency knobs
## External Boundary
Outputs (`audita.PolishResult`):
- polished transcript path
- optional report path
- generated config path
- stdout/stderr log paths
- exit code, duration, invoked binary
- adapter metadata
Narratio invokes `audita process` as a subprocess for each polish operation.
The configured timeout and parent cancellation bound the invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
## Boundaries
Owns:
- Deterministic CLI argument construction for `audita process`
- Environment bridging for API credentials
- Invocation config emission
- Output validation for polished transcript and report
## Request Contract
`PolishRequest` carries:
- required transcript/glossary/output/work-dir paths;
- optional report path (required when report mode is enabled);
- generated config and stdout/stderr log paths;
- optional per-invocation module override.
Does not own:
- Upstream/downstream stage orchestration
- Credential sourcing policy beyond required env-var presence check
The constructed runner owns static Audita settings: binary, timeout,
credentials, default modules, model and endpoint settings, validation and output
settings, report mode, and concurrency. The `polish` stage supplies only
invocation-specific paths and may override modules for that invocation.
## Config Fields Used
Via `pipeline.audita.*` mapped in app/stage wiring:
- `binary`, `timeout`, `llm_api_key_env`, `modules`, `base_url`, `model`
- `transcript_description`, `config_path`, `output_schema`, `work_dir_retention`
- `total_llm_concurrency`, `proposal_llm_concurrency`, `validation_model`, `validation_llm_concurrency`, `report`
## Result Contract
`PolishResult` returns:
- processed transcript path;
- optional report path;
- work dir and generated-config/log paths;
- exit code, duration, binary provenance;
- adapter metadata map.
## External Adapters Used
- Shared subprocess helper (`internal/adapters/subprocess`) to run CLI and capture logs.
## Validation and Failure Semantics
Construction fails for invalid static config values, including:
- empty binary;
- non-positive timeout;
- invalid base URL;
- invalid output schema;
- invalid work-dir retention value;
- invalid concurrency values.
## State and Manifest Behavior
- No direct manifest writes.
- Stage-level metadata records adapter provenance and credential-present signal.
- Generated invocation YAML is written when `GeneratedConfigPath` is provided.
Run fails for:
- missing required request paths;
- missing required credential env var when configured (`llm_api_key_env`);
- subprocess execution failure;
- invalid processed transcript JSON (`segments` array required);
- invalid report JSON when reporting is enabled.
## Skip and Resume Behavior
- Adapter has no skip/resume logic. Stage/runner controls this.
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
MiB. Both must be regular files without symlinked path components.
## Failure Behavior
- Constructor validation fails on invalid binary/timeout/schema/concurrency/URL values.
- Run fails on missing required paths, missing required credential env var, subprocess errors, invalid polished JSON shape, or invalid report JSON.
- Failures preserve stdout/stderr paths in returned result metadata.
Failure results still include output/log/config/exit metadata for diagnostics.
## Tests to Inspect Before Changing
- `internal/adapters/audita/subprocess_test.go`
- `internal/adapters/audita/fake_test.go`
- `internal/stage/polish_test.go`
## Deterministic Behavior
- CLI args are built from runner config + request in a fixed order.
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
- Manifest writes are stage-owned; adapter itself is stateless.
## Architectural Invariants
- Polished output must be valid JSON with top-level `segments` array.
- When report is enabled, report output must be valid JSON.
- If `llm_api_key_env` is configured, credential must be present in environment.
## Configuration
Operator-selected values are defined under `pipeline.audita.*` in the
[configuration reference](../config.md#pipeline).
Maintained example with Audita config:
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -0,0 +1,83 @@
# Notarius Integration Contract
## Boundary
Narratio uses Notarius as a subprocess to extract configured structured JSON
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
Canonical Notarius references:
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
records the exact current constraints for all ten D&D lanes. Treat the linked
Notarius documents as canonical when changing those values; Narratio does not
duplicate the complete schemas.
## Invocation
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
configuration path, input path, output directory, and working directory to
absolute paths and invokes:
```text
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
```
Standard output is reserved for the JSON receipt. Standard error is captured
separately as diagnostic output. Narratio applies the configured timeout and
does not interpret stdout as a receipt unless the subprocess exits successfully.
It does not pass a Narratio session ID or run `notarius config validate`
automatically; the configured working directory and Narratio's minimal child
environment apply to the subprocess.
## Accepted Result
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
must identify the configured pipeline, and its `index_file` must be exactly
`index.json` beneath the reported bundle root. The production index must name
the management files exactly as `manifest.json`, `rejected.json`, and
`warnings.json`. All receipt, index, and lane paths must stay inside that
bundle; symlinks and non-regular lane payloads are rejected.
Supported receipt and index shapes tolerate unknown fields for forward
compatibility, while required identity, validation, count, manifest,
rejection, warning, and lane-list fields remain mandatory. Narratio applies
bounded reads to the receipt, index, rejection, and warning documents. Optional
chunk-map and evidence-context descriptors must carry their complete generic
contract metadata when present.
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
index descriptor with the configured lane ID, media type, schema ID, schema
version, and, when configured, module key. Missing, duplicate, rejected, or
incompatible required lanes fail extraction even if Notarius exited zero.
Unconfigured lanes may remain in the preserved bundle but do not become
selectable Narratio sources.
Each accepted configured lane is registered as
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
resume validation but is not selectable. Scriptorium and publish rules consume
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
## Failure And Compatibility Behavior
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
- Unsafe or incompatible index data and required-lane rejection fail before the
staged bundle is promoted to durable storage.
- Contract and external provenance metadata are preserved on lane artifact
records and through explicit publication.
Rejection and warning summaries retain structured stage, scope, lane, and
reason-code fields for diagnostics without exposing free-form external messages
or reading lane payload bodies.
Configuration fields and defaults are in [Configuration](../config.md).
Operator paths, rerun procedures, and bundle retention are in
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
for failure recovery.

View File

@@ -1,64 +1,71 @@
# Integration: scriptorium
# Integration: Scriptorium
## Purpose
Define Narratio's adapter contract for Scriptorium artifact generation and render-debug subprocess invocations.
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
## Inputs and Outputs
Inputs:
- `RunArtifactRequest`: binary, config path, prompt/profile IDs, input map, vars map, timeout, output path, logs/config paths, optional API env and working dir
- `RenderArtifactRequest`: same core fields for render mode
## External Boundary
Outputs (`ArtifactResult`):
- output path
- stdout/stderr log paths
- generated config path
- exit code and duration
- command mode (`run` or `render`)
- prompt/profile provenance
- validation failure signal
- adapter metadata
Narratio invokes Scriptorium as a subprocess in these modes:
## Boundaries
Owns:
- Deterministic CLI arg construction for `scriptorium run` and `scriptorium render`
- Common request validation
- Invocation config emission
- Output existence/non-empty checks
- Validation-failure mapping for run exit code 2
- `scriptorium run`
- `scriptorium render`
Does not own:
- Artifact selection policy (`analyze` stage)
- Bounds semantic validation (`trim` stage)
The request timeout and parent cancellation bound each invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
## Config Fields Used
Via `pipeline.scriptorium.*` and stage-level artifact config:
- `binary`, `config_path`, `timeout`, `render_debug`
- artifact-level `prompt_id`, `profile_id`, `timeout`, `inputs`, `vars`, `output_path`
## Request Contract
Both request types carry:
- binary/config/prompt/profile IDs;
- input map and vars map;
- output path;
- timeout;
- generated config + stdout/stderr log paths;
- optional API-key env var name;
- optional working directory.
## External Adapters Used
- Shared subprocess helper (`internal/adapters/subprocess`).
## Result Contract
`ArtifactResult` returns:
- output/log/generated-config paths;
- exit code and duration;
- command mode (`run` or `render`);
- prompt/profile provenance;
- `ValidationFailed` marker;
- metadata map.
## State and Manifest Behavior
- No direct manifest writes.
- Stage metadata records adapter outputs and command mode.
- Generated invocation YAML is written when requested.
## Validation and Failure Semantics
Request validation fails for:
- missing binary, prompt id, or output path;
- non-positive timeout;
- empty input/var names;
- empty input path values;
- missing required credential env var when `APIKeyEnv` is set.
## Skip and Resume Behavior
- Adapter has no skip/resume logic. Stage/runner controls execution.
Run behavior:
- subprocess errors propagate with context;
- `run` exit code `2` is mapped to `ValidationFailed=true`;
- successful subprocess still fails if output file is missing or empty.
## Failure Behavior
- Request validation fails for missing binary/prompt/output, invalid timeout, invalid input/var names, or missing required API env var.
- Subprocess errors bubble with command context.
- `run` exit code 2 is treated as `ValidationFailed=true` and surfaced as error by calling stage.
- Successful subprocess still fails if output file is missing/empty.
Each artifact result is limited to 64 MiB and must be a regular file without
symlinked path components.
## Tests to Inspect Before Changing
- `internal/adapters/scriptorium/subprocess_test.go`
- `internal/adapters/scriptorium/fake_test.go`
- `internal/stage/analyze_test.go`
- `internal/stage/trim_test.go`
Render behavior:
- subprocess errors propagate;
- output file must exist and be non-empty.
## Architectural Invariants
- Both modes require explicit timeout > 0.
- Input/var maps are sorted into deterministic CLI argument order.
- Run-mode validation failures are represented explicitly, not silently skipped.
## Deterministic Behavior
- input and var maps are sorted into deterministic `--input` and `--var` CLI args.
- stage wiring adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream routing, overriding any configured `vars.session_id`.
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
- adapter is stateless and does not own artifact-selection policy.
## Configuration
Operator-selected values are defined under `pipeline.scriptorium.*`, including
per-artifact settings under `pipeline.scriptorium.artifacts.*`, in the
[configuration reference](../config.md#pipeline).
Maintained examples with Scriptorium config:
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -1,60 +1,62 @@
# Integration: seriatim
# Integration: Seriatim
## Purpose
Define Narratio's adapter contract for merge, normalize, and trim subprocess invocations of Seriatim.
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
## Inputs and Outputs
Inputs:
- `MergeRequest`: raw/per-speaker normalized transcript inputs, base output path, optional report, speaker/autocorrect paths, logs/config
- `NormalizeRequest`: input transcript, output path, schema, optional report, timeout/log/config
- `TrimRequest`: input transcript, output path, keep selector, timeout/log/config
## External Boundary
Outputs:
- `MergeResult`, `NormalizeResult`, `TrimResult` with output paths, logs/config paths, exit code, duration, binary provenance, and metadata.
Narratio invokes Seriatim as a subprocess in these modes:
## Boundaries
Owns:
- Validated deterministic CLI invocation construction
- Optional env tuning propagation for merge
- Invocation config file emission
- JSON output validation
- `seriatim merge`
- `seriatim normalize`
- `seriatim trim`
- `seriatim render`
Does not own:
- Transcript input selection/materialization logic (stage-owned)
- Bounds computation (scriptorium/trim-stage-owned)
The configured timeout and parent cancellation bound each invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
## Config Fields Used
Via `pipeline.seriatim.*` mapped in app/stage wiring:
- `binary`, `timeout`, `output_schema`, `coalesce_gap`, `report`
- `env.overlap_word_run_gap`
- `env.overlap_word_run_reorder_window`
- `env.backchannel_max_duration`
- `env.filler_max_duration`
## Request/Result Contracts
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
- `TrimRequest`/`TrimResult`: transcript trimming with required keep selector.
- `RenderRequest`/`RenderResult`: transcript-to-markdown rendering with explicit format and render booleans.
## External Adapters Used
- Shared subprocess helper (`internal/adapters/subprocess`).
Results include output/log/config paths, timing, exit code, and metadata.
## State and Manifest Behavior
- No direct manifest writes.
- Stage metadata consumes adapter result fields and preserves generated config/log references.
## Validation and Failure Semantics
Runner construction validates:
- binary presence;
- timeout > 0;
- supported output schema (`seriatim-minimal|seriatim-intermediate|seriatim-full`);
- non-negative coalesce gap.
## Skip and Resume Behavior
- Adapter has no skip/resume logic. Runner controls stage execution.
Invocation fails on:
- missing required request paths/inputs;
- invalid normalize schema override;
- unsupported render format;
- subprocess failure;
- invalid JSON outputs for merge/normalize/trim;
- missing `segments` array for normalize/trim transcript outputs;
- empty render output files.
## Failure Behavior
- Constructor fails for invalid binary/timeout/output-schema/coalesce-gap.
- Merge fails on missing output path/inputs/report path (if enabled), subprocess errors, invalid merged output JSON, invalid report JSON.
- Normalize fails on missing input/output, invalid schema, subprocess errors, invalid final output JSON shape, invalid report JSON.
- Trim fails on missing input/output/keep selector, subprocess errors, invalid final-trimmed output JSON shape.
When report paths are provided/enabled, report files must parse as JSON.
Each Seriatim JSON or rendered-text result is limited to 64 MiB and must be a
regular file without symlinked path components.
## Tests to Inspect Before Changing
- `internal/adapters/seriatim/subprocess_test.go`
- `internal/adapters/seriatim/fake_test.go`
- `internal/stage/merge_test.go`
- `internal/stage/normalize_test.go`
- `internal/stage/trim_test.go`
## Deterministic Behavior
- argument ordering is deterministic per command construction.
- merge env overrides are explicit (`SERIATIM_*`) and only emitted when configured.
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
- adapter does not write manifests or choose stage inputs.
## Architectural Invariants
- Supported output schemas are limited to `seriatim-minimal`, `seriatim-intermediate`, `seriatim-full`.
- Final and final-trimmed outputs must include `segments` arrays.
- Merge/normalize/trim all route through deterministic subprocess invocation.
## Configuration
Operator-selected values are defined under `pipeline.seriatim.*` and
`pipeline.render.*` in the
[configuration reference](../config.md#pipeline).
Maintained examples with Seriatim config:
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -0,0 +1,71 @@
# Integration: WhisperX
## Purpose
WhisperX transcribes each prepared speaker audio file for Narratio's
`transcribe` stage. Narratio uses an HTTP boundary and installs each successful
response as that speaker's raw transcript JSON.
## HTTP Boundary
Narratio sends an HTTP `POST` to the configured transcription URL using
`multipart/form-data` with:
- `file`: the audio file, retaining its base filename; and
- `language`: the configured language string.
The server must return a `2xx` response whose body is valid JSON. Narratio does
not currently require a more specific response schema at this boundary.
The transcription URL must be an absolute `http` or `https` URL. The audio body
is streamed through a fresh multipart writer for every attempt, so its memory
use is bounded by the transport buffer rather than by the complete audio file.
WhisperX response acquisition is capped at 10 MiB.
## Request And Result Contract
Each adapter request identifies a speaker, a readable audio file, and the
destination for the raw transcript. The HTTP request carries the audio and
language; the speaker identifier remains Narratio orchestration metadata.
On success, Narratio atomically writes the response body to the requested
destination. The adapter result reports that logical output together with the
attempt count, final HTTP status when available, elapsed duration, and adapter
identity metadata. A failed or invalid response is not installed as the
transcript output.
## Retry, Timeout, And Cancellation
- The configured timeout applies independently to each HTTP attempt.
- `retries` means additional attempts after the first.
- HTTP `429`, HTTP `5xx`, attempt timeouts, and network errors are retryable.
- Other HTTP `4xx` responses and explicit cancellation are not retryable.
- Narratio waits the configured retry delay between attempts and aborts that
wait when the parent context is canceled.
## Validation And Failure Semantics
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
a missing language, a non-positive timeout, negative retries, or a negative
retry delay. A request fails before transmission when its audio or output path
is missing.
Non-`2xx` status, transport failure, response-size overflow, invalid JSON, or
failure to install the output causes the transcription to fail. Errors include
attempt context, and the result retains attempts, final status when available,
and elapsed duration for diagnostics.
## Determinism And Concurrency
Each audio request has stable multipart field names, and successful bytes are
installed atomically. The transcribe stage may process speaker files in
parallel, bounded by the configured concurrency. It records results in stable
speaker order after all work completes; any speaker failure fails the stage.
## Related Canonical Docs
- [Configuration](../config.md#pipeline) defines the operator-selected
WhisperX URL, language, timeouts, retry policy, and concurrency.
- [Adapter implementation](../internal/adapters.md) describes internal wiring.
- [Transcribe stage](../internal/stage-transcribe.md) describes stage mechanics,
durable artifacts, and manifests.

View File

@@ -1,29 +0,0 @@
# Internal Documentation Index
## Audience
Developers and LLM coding agents changing Narratio internals.
## Scope
Implementation-accurate contracts for workspace/state, manifests, stages, artifact resolution, adapter boundaries, and restore command behavior.
## Component Docs
- `adapters.md`: external adapter map, runtime wiring, and boundary ownership.
- `storage.md`: remote storage backend contracts and object-store invariants.
- `manifest.md`: session/run manifest schemas, lifecycle transitions, and persistence semantics.
- `artifacts.md`: built-in artifact registry, runtime artifact catalog, and source-resolution behavior.
- `workspace.md`: local state model, manifests, run-local layout, materialization, and cleanup invariants.
- `command-restore.md`: restore command discovery/planning/execution/reporting contract.
- `stage-prepare.md`: input materialization and provenance capture.
- `stage-transcribe.md`: WhisperX transcript generation.
- `stage-merge.md`: Seriatim normalization + merge.
- `stage-polish.md`: Audita transcript polishing.
- `stage-normalize.md`: post-polish normalization.
- `stage-trim.md`: bounds-driven transcript trimming.
- `stage-analyze.md`: dependency-ordered Scriptorium artifact generation for selected configured artifacts.
- `stage-publish.md`: publish upload and current-pointer commit contract.
## External Integration Notes
- `../integrations/README.md`: canonical location for external integration contracts (`audita.md`, `seriatim.md`, `scriptorium.md`).
## Canonical Owner
`docs/internal/` is the canonical home for implemented internals per `docs/documentation/policy.md`.

View File

@@ -1,78 +1,90 @@
# Internal: Adapters
## Purpose
Describe the external adapter boundaries used by Narratio stages and app orchestration, including default runtime wiring.
## Inputs and outputs
Inputs:
- Stage requests passed through adapter interfaces (for example transcription, merge/normalize/trim, polish, artifact generation, object-store operations, notifications).
- Resolved config values used to construct default adapters.
Explain the adapter interfaces and production composition used by application
and stage orchestration. Externally observable protocols and formats belong in
the [integration contracts](../integrations/).
Outputs:
- Adapter-specific result structs (paths, metadata, status/attempt info, duration/exit details).
- Adapter errors returned to stage/app orchestration.
## Adapter Boundaries
## Boundaries
Owns:
- Transport/process/SDK details at system boundaries (`HTTP`, subprocess CLI invocation, AWS SDK calls).
- Request/response contracts in `internal/adapters/*` packages.
Narratio stage logic depends on adapter interfaces, not transport-specific details.
Does not own:
- Stage sequencing, skip/force/resume decisions.
- Manifest transition logic.
- Canonical workspace path policy.
Primary adapters:
## Config fields used
Default wiring and adapter calls consume:
- `pipeline.whisperx.*`
- `pipeline.seriatim.*`
- `pipeline.audita.*`
- `pipeline.scriptorium.*`
- `pipeline.storage.*` and `pipeline.publish.*` (object-store construction/gating)
- `pipeline.notification.*` (sender boundary exists; placeholder behavior today)
## External adapters used
Runtime env boundary fields (`internal/stage.Env`):
- `whisperx.Client`
- `seriatim.Runner`
- `audita.Runner`
- `scriptorium.Runner`
- `notarius.Runner`
- `storage.ObjectStore`
- `notify.Sender`
Current execution usage:
- Actively used by implemented stages: `WhisperX`, `Seriatim`, `Audita`, `Scriptorium`, `ObjectStore`, `Notifier`.
- Present but not used by implemented stage set: legacy `storage.Backend`.
## Ownership
Default construction in app runner:
- Auto-constructed when not injected: WhisperX HTTP client, Seriatim subprocess runner, Audita subprocess runner, Scriptorium subprocess runner, object store (only when needed), and `notify.NoopSender`.
- Object-store construction goes through app command orchestration so configured filesystem secrets are loaded before the storage adapter is initialized.
- Callers can inject test/fake implementations through `app.RunOptions.Env`.
Adapters own:
## State and manifest behavior
- Adapters do not directly mutate session/run manifests.
- Stages and runner own manifest writes and stage status transitions.
- Adapter outputs are persisted indirectly through stage result mapping (outputs/logs/generated configs/metadata).
- HTTP/subprocess/SDK argument and transport details.
- Backend-specific request/response mapping.
## Skip and resume behavior
- No adapter-level skip/resume semantics.
- Skip/resume/force behavior is decided by app runner using manifest stage state.
Adapters do not own:
## Failure behavior
- Adapter constructors validate config-derived values and fail early on invalid required inputs.
- Adapter run-time failures are returned to stage code with boundary context and are recorded as stage failures by runner logic.
- Subprocess adapters preserve stdout/stderr and generated-config paths to aid diagnosis.
- stage ordering/skip/force logic;
- manifest transitions;
- canonical path policy.
## Tests to inspect before changing
## Default Wiring
`internal/app/runner.go` initializes default adapters when not injected:
- WhisperX HTTP client from pipeline config.
- Seriatim subprocess runner.
- Audita subprocess runner.
- Scriptorium subprocess runner.
- Notarius subprocess runner when extraction is enabled.
- Noop notifier (`notify.NoopSender`).
- Object store only when required by selected stages/config.
Notarius is composed only when extraction is enabled; the extract stage owns
receipt, bundle, and configured-lane policy rather than the adapter.
Object-store construction goes through `newCommandObjectStore`, which loads
configured filesystem secrets before adapter initialization.
## Failure Semantics
- Constructor errors fail stage execution setup early.
- Runtime adapter errors propagate to stage code and then manifest failure handling.
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
- Shared subprocess execution starts an owned process group on Linux/macOS or a
kill-on-close job object on Windows. Every terminal path disposes of that
owned tree before returning. After a natural leader exit, Unix checks for
remaining group members and uses bounded graceful then forceful termination;
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
diagnostic limits use the same terminal disposal path without losing their
original result classification. Child environments contain only the execution
baseline and adapter-specified values; configured credentials are explicit
sensitive values. Stdout and stderr are redacted while streaming into separate
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
departed leader's descendant. Unsupported platforms reject owned command
execution.
## Implementation And Tests
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
- Shared subprocess mechanics: `internal/adapters/subprocess`
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
- `internal/adapters/whisperx/http_test.go`
- `internal/adapters/seriatim/subprocess_test.go`
- `internal/adapters/audita/subprocess_test.go`
- `internal/adapters/scriptorium/subprocess_test.go`
- `internal/adapters/notarius/subprocess_test.go`
- `internal/adapters/storage/*_test.go`
- `internal/adapters/notify/fake_test.go`
- `internal/app/runner_test.go`
## Architectural invariants
- Stage code depends on adapter interfaces, not transport-specific implementation types.
- External SDK-specific types remain inside adapter implementations.
- Default app wiring must remain deterministic and overrideable via injected env dependencies.
See the [WhisperX](../integrations/whisperx.md),
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
[Scriptorium](../integrations/scriptorium.md), and
[Notarius](../integrations/notarius.md) contracts before changing an
externally visible boundary. Operator-selected values belong in
[Configuration](../config.md).

View File

@@ -1,106 +1,214 @@
# Internal: Artifacts
## Purpose
Define Narratio artifact identity, catalog, and source-resolution behavior for:
- built-in session artifacts;
- configured analyze artifacts;
- canonical previous-session artifact sources.
## Inputs and outputs
Inputs:
- configured input sources (`pipeline.scriptorium.artifacts.*.inputs.*.source`);
- session paths and manifest inputs/outputs;
- runtime catalog state.
Explain the artifact registry, runtime catalog, resolver, previous-input
requirements, and shared remote current-state mechanics implemented by
`internal/artifacts`. Configuration fields that accept source IDs belong in
[Configuration](../config.md); physical placement belongs in
[Operations](../operations.md).
Outputs:
- resolved artifact path + provenance (`ResolvedSessionArtifact`);
- runtime catalog entries for built-ins and configured artifacts;
- requirement sets for canonical previous-session inputs;
- canonical S3 session, run, current, session config, session locks, audio, and published output keys.
## Built-in Source IDs
## Boundaries
Owns:
- built-in source registry and validation;
- configured artifact catalog identity (`narratio.artifact.<name>`);
- canonical previous-session source parsing and resolution;
- previous-session requirement collection (`CollectPreviousArtifactRequirements`).
The internal registry recognizes these stable built-in source IDs:
Does not own:
- prepare-stage remote hydration;
- stage success/skip transitions;
- publish upload orchestration.
- `narratio.transcript.base`
- `narratio.transcript.polished`
- `narratio.transcript.final`
- `narratio.transcript.final_trimmed`
- `narratio.transcript.final_markdown`
- `narratio.transcript.final_trimmed_markdown`
- `narratio.bounds.session`
## Built-in IDs
| Artifact ID | Canonical file | Producer stage | Output kind |
| --- | --- | --- | --- |
| `narratio.transcript.base` | `transcripts/base.json` | `merge` | `transcript_base` |
| `narratio.transcript.polished` | `transcripts/polished.json` | `polish` | `transcript_polished` |
| `narratio.transcript.final` | `transcripts/final.json` | `normalize` | `transcript_final` |
| `narratio.transcript.final_trimmed` | `transcripts/final.trimmed.json` | `trim` | `transcript_final_trimmed` |
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` |
Registry entries bind each ID to its producer, output kind, canonical fallback,
and content validator. The focused stage documents own their input/output flow;
[Configuration](../config.md) owns where operators may select these IDs.
## Source families
- built-in: `narratio.transcript.*`, `narratio.bounds.session`
- configured artifact: `narratio.artifact.<artifact_key>`
- canonical previous-session artifact: `narratio.previous_session.artifact.<artifact_key>`
## Configured, Extraction, And Previous-Session Sources
## S3 key helpers
- session prefix: `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
- session config: `{session_prefix}/session.yml`
- session lock store: `{session_prefix}/locks.yml`
- run prefix: `{session_prefix}/runs/{run_id}/`
- audio prefix: `{session_prefix}/{session.inputs.audio_s3.prefix}`
- current manifest: `{session_prefix}/current/manifest.json`
- current run pointer: `{session_prefix}/current/run_id.txt`
- configured source ID format: `narratio.artifact.<artifact_key>`
- extraction source ID format: `narratio.extraction.<output_key>`
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
## Runtime catalog model
Catalog entries track:
- `planned`: source is registered for this run;
- `executable`: configured artifact is selected for analyze execution;
- `available`: usable local file exists (generated this run or reused from disk).
All formats are validated by strict source-policy rules. Configured artifact and
extraction keys use `^[a-z][a-z0-9_]*$`; source parsers never normalize an
unrecognized token into a valid source. Extraction sources are registered only
from `pipeline.notarius.outputs`; the Notarius index has no selectable source
ID.
## Runtime Catalog
`ArtifactCatalog` tracks:
- `planned`: source registered for run context;
- `executable`: included in the effective analyze artifact set;
- `available`: local file exists and validates;
- `provenance`: availability source.
Configured definitions are always registered. Without an explicit selection,
the effective analyze set contains enabled definitions. With `--artifacts`, the
exact named configured definitions become the effective set for that invocation,
regardless of their `enabled` value; dependencies are not added implicitly.
Availability is separate from executability: a non-executable configured output
may be reused from a canonical non-empty file, while an executable definition
is generated by analyze. Extraction entries are registered from configuration
and become available only after compatible extraction evidence is hydrated.
Current provenance values:
Configured artifact provenance values include:
- `generated.current_analyze_run`
- `filesystem.disabled_artifact_output`
Previous-session canonical provenance values include:
- `manifest.inputs.previous_cache`
- `current_session.previous_cache`
## Resolution behavior
- Built-ins resolve via manifest producer outputs first, then canonical fallback paths.
- Configured `narratio.artifact.<name>` sources resolve through catalog availability.
- Canonical previous-session sources resolve to current-session `previous/` cache candidates derived from configured artifact canonical output paths.
- Publish-relative configured artifact paths under `artifacts/` are cached without a redundant nested `artifacts/` segment.
- Previous-session canonical resolution prefers manifest-recorded input paths when present, then filesystem fallback under `previous/artifacts/**`.
## Resolution Rules
## Previous-session requirement scanning
`CollectPreviousArtifactRequirements`:
- scans enabled configured artifacts only;
- includes canonical previous-session sources only;
- deduplicates by artifact key;
- merges required/optional references (`required` wins);
- records deterministic sorted source locations for diagnostics.
Built-ins:
## Validation behavior
- transcript built-ins: JSON with top-level `segments` array;
1. manifest producer outputs (when present)
2. canonical session-path fallback
Configured sources (`narratio.artifact.*`):
- resolve only through runtime catalog availability.
Extraction sources (`narratio.extraction.*`):
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
- require a current successful extract record with the exact configured source,
compatible contract and Notarius provenance, a confined regular durable
payload, matching checksum, and the current resolved trimmed-transcript
identity;
- remain unavailable unless catalog hydration receives valid evidence. Resume
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
an error; and
- are never inferred by scanning the Notarius bundle directory.
Previous-session sources (`narratio.previous_session.artifact.*`):
- resolve only from local `previous/` cache state;
- prefer manifest-backed previous-input paths;
- fallback to existing previous-cache filesystem paths.
Source absence is evaluated by the consuming artifact input. An optional input
is omitted from that invocation; a required input fails resolution. This is
separate from a stage's lifecycle outcome.
Validation by content type:
- transcript JSON built-ins: JSON with top-level `segments` array;
- transcript Markdown built-ins: non-empty text file;
- bounds built-in: valid JSON;
- configured and previous-session artifact files: non-empty text content.
- configured/previous-session artifact files: non-empty text file.
## Failure behavior
- unsupported source or malformed canonical previous source: validation/resolution error;
- known source unavailable: `ErrSessionArtifactNotFound`;
- configured/previous canonical source without catalog: error;
- resolved invalid file content: validation error.
## Previous Requirement Collection
## Tests to inspect before changing
- `internal/artifacts/artifact_resolver_test.go`
- `internal/artifacts/catalog_test.go`
- `internal/artifacts/previous_requirements_test.go`
- `internal/stage/prepare_previous_test.go`
- `internal/stage/analyze_test.go`
`CollectPreviousArtifactRequirements`:
## Architectural invariants
- Built-in source IDs are static.
- Configured and previous-session source IDs are artifact-key based and validation-gated.
- Resolution behavior remains deterministic and manifest-aware.
- scans the effective configured artifact set;
- extracts only canonical previous-session sources;
- deduplicates by artifact key;
- merges required and optional references (required wins);
- returns deterministic ordering and source locations.
## Current-State Helpers
Artifacts package owns shared remote current-state loading mechanics used by
restore, status and validation checks, and previous-cache planning.
For a new-protocol current state, the pointer-selected immutable commit is the
complete restore authority. Callers receive its declared object identities and
must not supplement them by listing mutable session prefixes. The legacy reader
is intentionally separate and remains migration-only support.
The reader opens each small control object directly and enforces owner-specific
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
manifest limit to `current/manifest.json`. These are exposed as
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
`MaxLegacyCurrentManifestBytes`.
Each read uses the generation and size metadata returned with its opened body.
Actual bytes remain subject to a limit-plus-one read even if size metadata is
absent or inaccurate. Immutable selections then retain their declared-size,
checksum, generation, and identity checks. No current-state control object is
downloaded through a temporary file.
Core helpers:
- `LoadCurrentState`
- `ValidateCurrentStateIdentity`
- `RemoteCommitManifest` and `CurrentCommitPointer`
Typed missing-state errors:
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
Identity validation supports caller-provided expectations:
- expected campaign;
- expected session ID;
- expected run ID, or pointer/manifest run-ID consistency check.
Caller policy is intentionally outside artifacts helpers:
- some callers fail on missing current state;
- some callers downgrade missing state to status/findings;
- some callers skip optional behavior when state is missing.
## Key Path Helpers
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
- session/work/run paths;
- previous-cache paths;
- spool/cache paths;
- S3 session/run/current-state key layout.
New publication creates run-scoped immutable objects, including
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
mutable selector is `current/commit-pointer.json`; readers verify its selected
commit and declared object generations/checksums. Legacy current-pair loading
is confined to `current_state_legacy.go` for migration only.
Campaign, session, and Narratio run IDs are validated as portable opaque
segments at configuration and artifact boundaries before they can be used in a
workspace or S3 namespace. Previous-artifact destinations remain typed,
multi-segment relative paths and are confined beneath `previous/artifacts`; they
are not treated as opaque identifiers.
See [Workspace Internals](workspace.md) for how callers consume local helpers
and [Operations](../operations.md#local-state-layout) for the authoritative
physical layout.
## Invariants
- source ID formats are stable contracts;
- artifact resolution is deterministic and manifest-aware;
- extraction sources are available only from a compatible successful manifest
record;
- previous-session source resolution in `analyze` is local-only;
- remote current-state key construction remains centralized in artifacts helpers.
## Implementation And Tests
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
`internal/artifacts/extraction_catalog.go`,
`internal/artifacts/extraction_evidence.go`,
`internal/artifacts/extraction_input.go`
- Current state: `internal/artifacts/current_state.go`,
`internal/artifacts/current_state_commit.go`,
`internal/artifacts/current_state_legacy.go`
- Paths and keys: `internal/artifacts/paths.go`,
`internal/artifacts/s3_keys.go`
- Previous requirements: `internal/artifacts/previous_requirements.go`
- Tests: `internal/artifacts/artifact_resolver_test.go`,
`internal/artifacts/catalog_test.go`,
`internal/artifacts/extraction_catalog_test.go`,
`internal/artifacts/current_state_test.go`,
`internal/artifacts/paths_model_test.go`,
`internal/artifacts/previous_requirements_test.go`

View File

@@ -1,106 +1,110 @@
# Internal: Command Restore
## Purpose
Define the implemented `narratio session restore` contract: committed remote-state discovery, deterministic plan classification, safe file install semantics, and restore reporting.
## Inputs and outputs
Inputs:
- CLI syntax: `narratio session restore <session_id>`.
- CLI flags: `--config`, `--campaign`, `--campaign-file`, `--session`, `--previous-session-id`, `--dry-run`, `--force`, `--include-audio`.
- Resolved/validated `pipeline.yml` and `session.yml`.
- Configured remote object store.
- Remote committed current-state markers (`current/run_id.txt`, `current/manifest.json`).
Explain the implemented restore discovery, planning, installation, and
reporting flow in `internal/app`. User invocation belongs in
[CLI](../cli.md#session-restore), and the operator recovery procedure and
physical restore scope belong in
[Operations](../operations.md#restore-workflow).
Outputs:
- Dry-run summary to stdout (plan + counts).
- Non-dry-run completion summary to stdout.
- Local durable session files restored under canonical session root.
- Non-dry-run restore report at `reports/restore-latest.json`.
Restore separates remote authority, local conflict policy, and filesystem
mutation so each remains testable independently.
## Boundaries
Owns:
- Restore command flag parsing and command wiring.
- Remote current-state discovery and identity validation.
- Restore plan construction and conflict classification.
- Restore execution for planned downloads.
- Restore report model and persistence.
## Discovery Contract
Does not own:
- Stage execution orchestration (`run`, `resume`, `run-stage`).
- Publish-stage behavior.
- Storage transport implementation details (owned by storage adapters).
Discovery delegates current-state pointer and manifest loading to
`internal/artifacts`, then validates the result against the resolved request:
## Config fields used
- Config/session discovery and templating fields consumed by all commands.
- `pipeline.workspace.root` (local restore target root).
- `pipeline.storage.*` (remote backend + publish identity derivation).
- `pipeline.storage.s3.*` identity components used by session-prefix helpers.
- `pipeline.spool.root` for active audio downloads.
- `pipeline.cache.root` and `pipeline.cache.s3_audio` for reusable S3 audio cache.
- `session.session_id`
- `session.campaign`
- campaign must match;
- session ID must match.
- run ID must match the pointer-selected committed run.
## External adapters used
- `storage.ObjectStore` for `Exists`, `List`, `Download`.
- `artifacts.Store` (`LocalStore`) for layout and session lock management.
- `manifest.LocalStore` for manifest decode/validation and identity checks.
Restore treats any missing or invalid remote current state as a command error.
## State and manifest behavior
- Restore is not a pipeline run and does not create a run manifest.
- Restore uses committed remote current state only:
- `current/run_id.txt` must exist and be non-empty.
- `current/manifest.json` must decode and match requested session/campaign.
- Non-dry-run writes restore files to canonical session paths.
- With `--include-audio`, restore uses the shared S3 audio cache for `audio/**` objects. Cache hits avoid object downloads; cache misses download through spool, install the work file, and populate cache.
- Manifest install behavior:
- validated before replacement.
- installed last among download actions.
- existing local manifest is preserved if restored manifest validation/install fails.
- Non-dry-run report persists summary/action status metadata in `reports/restore-latest.json`.
## Planning Contract
Restore path scope:
- includes:
- `manifest.json`
- `transcripts/**`
- `artifacts/**`
- `previous/**`
- `audio/**` only when `--include-audio` is set
- excludes:
- `runs/**`
- `logs/**`
- `reports/**`
- `config/**`
- `inputs/**`
- remote `current/**` pointer files as local restore targets
Restore planner action kinds:
## Skip and resume behavior
- Restore does not participate in stage skip/resume decisions.
- Restore provides durable local state so subsequent stage commands can resume or rerun based on restored manifest state.
- Audio cache is outside the workspace and is reused across restore and prepare invocations.
- Dry-run is read-only and returns plan output only.
- `download`;
- `skip_same`;
- `conflict`.
## Failure behavior
- Fails when storage backend is unavailable or publish identity cannot be resolved.
- Fails when remote current pointer/manifest is missing or invalid.
- Fails when remote manifest identity mismatches requested campaign/session.
- Fails on local conflicts unless `--force` is set.
- Fails fast on session lock acquisition conflict for non-dry-run execution.
- On execution failure, previously installed files remain; no rollback is performed.
Planner behavior:
## Tests to inspect before changing
- `internal/app/restore_test.go`
- `internal/app/restore_discovery_test.go`
- `internal/app/restore_plan_test.go`
- `internal/app/restore_execution_test.go`
- `internal/app/restore_workflow_test.go`
- `internal/artifacts/archive_identity_test.go`
- a new-protocol restore uses only the selected commit's declared artifact set;
each action carries that artifact's immutable key, checksum, size, and
generation. Coherent legacy state remains on the isolated compatibility path;
- remote-to-local mapping is traversal-safe;
- actions are sorted by local relative path and then remote key;
- force converts differing eligible regular files from conflicts to downloads;
directories and other non-regular targets remain conflicts.
## Architectural invariants
- Restore relies on centralized path/key helpers (`internal/artifacts`) rather than ad hoc key building.
- `current/run_id.txt` is the remote commit marker; restore must not infer committed state from incidental files.
- Local path mapping is traversal-safe and constrained to session root.
- Restore scope is deterministic and path-classified:
- include `manifest.json`, `transcripts/**`, `artifacts/**`, `previous/**`
- include `audio/**` only with `--include-audio`
- exclude `runs/**`, `logs/**`, `reports/**`, `config/**`, `inputs/**`
- Command remains standalone; no implicit `run --restore` behavior.
For a non-dry-run restore, planning/classification happens only after acquiring
the session lock. Runner manifest/reuse checks acquire that same lock first.
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
prepare, status, and validation. A committed source is selected only by its
exact source identity; legacy fallback remains isolated and rejects ambiguity.
## Execution Contract
Execution order and safety:
- non-manifest downloads happen before manifest install;
- `manifest.json` installs last;
- downloads use sibling temp files plus atomic rename;
- manifest replacement is validated before rename;
- each committed object is verified against its declared checksum, size, and
generation before installation;
- a committed manifest already verified during discovery is retained for the
matching restore action and revalidated before installation, avoiding a
second body transfer;
- failed installs do not roll back files already written in the same execution.
- a durable `.restore-incomplete.json` marker is written before installation.
It blocks runners until a restore retry completes all verified installs and
the local manifest replacement, at which point it is removed.
- restored manifest local references are rebased beneath the selected local
session root. Unsafe relative references and producer-machine absolute paths
outside the manifest's producer session root are rejected; producer-local
spool/cache and cleanup locations are not restored as authority.
Audio restore path:
- uses `audio.MaterializeS3Audio`;
- integrates spool and S3 audio cache paths;
- reuses cached audio only when its no-follow regular file, content digest, and
identity sidecar all match the selected remote object version; otherwise it
refreshes through the durable download path.
## Reporting Contract
- dry-run mode prints a summary, performs no durable session writes, and may
read remote current-state or object-identity data to produce that summary;
- execution mode persists the canonical restore report described in
[Operations](../operations.md#restore-workflow);
- report includes plan counts, per-action status, and execution failures.
## Invariants
- restore uses committed remote current state as authority;
- one restore or status inspection observes the single pointer-selected commit
loaded at discovery; later pointer changes cannot add objects or substitute a
different run into its plan;
- a verified `current/commit-pointer.json` and its selected immutable commit
establish new-protocol remote commitment; coherent legacy
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
support;
- restore does not execute pipeline stages.
## Implementation And Tests
- Discovery: `internal/app/restore_discovery.go`
- Planning: `internal/app/restore_plan.go`, `internal/previouscache`
- Execution: `internal/app/restore_execute.go`
- Reporting and command coordination: `internal/app/restore_report.go`,
`internal/app/restore.go`
- Tests: `internal/app/restore_discovery_test.go`,
`internal/app/restore_plan_test.go`,
`internal/app/restore_execution_test.go`,
`internal/app/restore_workflow_test.go`

71
docs/internal/fileops.md Normal file
View File

@@ -0,0 +1,71 @@
# Internal: File Operations
`internal/fileops` owns the narrow mechanics for durable replacement of one
byte file. Callers keep ownership of serialization, validation, cancellation,
and destination-directory policy.
## Destination Confinement
Before it creates, replaces, or installs a destination file, `fileops` opens
each ancestor from the filesystem root and rejects symbolic links or components
that change during traversal. The resulting parent-directory handle is retained
for sibling temporary-file creation and rename, so a later pathname swap cannot
redirect the replacement. Existing destination symlinks are replaced as leaf
entries; their targets are never followed.
Remote object acquisition uses a writer supplied by the storage owner. The
writer receives a `fileops`-owned, already-open sibling temporary file rather
than a mutable destination path. Callers still own remote object selection,
validation, conflict handling, and final mode.
Directory promotion keeps the verified destination parent open while it creates
the temporary tree, copies regular source entries, and performs the platform
no-replace rename. Platforms without a verified handle-relative atomic
no-replace primitive reject promotion before writing a temporary tree.
## Cleanup Contract
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
the root and each target ancestor without following symlinks, then removes the
tree through those directory handles. It rejects root deletion and any symlink
encountered in the target path or tree; repeated removal of a missing target is
successful. Command and post-publish policy remains owned by `internal/app`.
## Confined Reads
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
caller-selected root and relative file path; `ReadRegularFile` is its
path-based convenience wrapper. They verify every ancestor through directory
handles and admit only a stable regular-file handle. Callers enforce their own
byte limits and access policy. Credential mode policy and environment
precedence remain owned by `internal/app`.
## Replacement Contract
`ReplaceFileAtomic` requires an existing destination directory. It creates a
sibling temporary file, writes the complete byte sequence, applies the
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
check, replaces the destination with a rename, then syncs the containing
directory.
The pre-rename check is the last point at which a caller can cancel without
installing a new destination. A failure before the rename leaves the old
destination unchanged and removes the temporary file; any cleanup failure is
returned alongside the primary failure. A failure after the rename may leave
the new file visible, but it is not reported as crash-durable.
Replacement follows the operating system's same-filesystem rename semantics.
If a platform cannot replace an existing destination, the operation returns an
error and never removes the old file as an emulation step.
## Directory-Sync Support
Linux and macOS attempt to sync the destination directory. Windows opens the
directory with backup semantics and flushes its buffers. If either operation
is unavailable for the platform, directory handle, or filesystem,
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
as successful crash-durable replacement.
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
retain their compatibility behavior of creating the destination parent with
the repository's workspace permissions before using this contract.

View File

@@ -1,81 +1,157 @@
# Internal: Manifest
## Purpose
Describe Narratio's durable execution state model for session-level and run-level manifests, including lifecycle transitions and persistence behavior.
## Inputs and outputs
Inputs:
- Session identity and run identity from app orchestration.
- Stage transition events and stage result payloads.
Explain the session-progress and invocation-audit models implemented by
`internal/manifest`. Physical manifest placement belongs in
[Operations](../operations.md#local-state-layout).
Outputs:
- Session manifest at `{workspace.root}/work/{campaign}/{session_id}/manifest.json`.
- Run manifest at `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`.
## Session Manifest
## Boundaries
Owns:
- Manifest schemas (`Manifest`, `RunManifest`, stage records, error records, input/artifact records).
- Stage status/action transition methods.
- Persistent store contract (`manifest.Store`) and local JSON store implementation.
`manifest.Manifest` records:
Does not own:
- Stage implementation details.
- Path construction policy outside manifest file persistence calls.
- CLI command behavior.
- identity (`session_id`, `campaign`, `run_id`)
- local path metadata (`local_workdir`, `local_spool_dir`)
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
- `inputs` records
- durable `artifacts` records
- per-stage `stages` map
- an optional `post_publish_cleanup` obligation, which binds a committed run,
remote commit identity, and each exact root-confined local target to its
completion evidence
## Config fields used
Manifest package itself does not read config directly.
Session, campaign, and run identities in local and downloaded manifests must be
portable opaque segments. Unsafe legacy identities are rejected with migration
guidance rather than being normalized into a different workspace or remote
namespace.
Manifest identity fields are populated by app/stage orchestration from:
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.storage.s3.*` (when publish/S3 identity is set)
The model admits these stage states:
## External adapters used
- No external service adapters.
- Uses local filesystem for persistence via `manifest.LocalStore`.
- `pending`
- `running`
- `succeeded`
- `failed`
- `skipped`
- `stale`
- `interrupted`
## State and manifest behavior
Session manifest model:
- Tracks durable per-session stage state and provenance (`pending`, `running`, `succeeded`, `failed`, `skipped`, `stale`, `interrupted`).
- Stores resolved inputs, durable artifacts, stage logs/config refs, and stage metadata.
## Run Manifest
Run manifest model:
- Tracks one invocation (`run_id`) with requested stages and force mode.
- Tracks per-stage action (`run` or `skip`) and per-stage status.
- Tracks overall run status (`running`, `succeeded`, `failed`).
`manifest.RunManifest` is created for each invocation and records:
Persistence behavior:
- Load validates required identity/timestamp fields and normalizes maps/records.
- Save updates `updated_at` and writes JSON atomically (temp file + rename).
- Session and run manifests are saved incrementally before/after stage transitions.
- invocation identity and `force` flag
- requested stages
- per-stage action (`run` or `skip`)
- per-stage status
- overall run status (`running`, `succeeded`, `failed`)
Relationship during execution:
- Runner updates both manifests for every stage transition.
- Session manifest is the durable pipeline-progress ledger.
- Run manifest is invocation history and audit record.
- Analyze stage outputs are persisted as `kind=scriptorium_artifact` with `source_id=narratio.artifact.<name>` for configured artifact identity.
## Remote Commit Manifest
## Skip and resume behavior
- Resume and skip decisions are based on session-manifest stage statuses.
- `--force` reruns selected stages and marks downstream succeeded stages as `stale` in session manifest.
- Run manifest records whether each stage was executed or skipped in that invocation.
`artifacts.RemoteCommitManifest` is a separate, versioned remote snapshot
contract. It is not a serialized session manifest and contains no local
post-publication assertion such as `current_pointer_written`. A remote commit
identifies one campaign, session, and run and declares its immutable artifact
set. Each artifact has a typed source, immutable destination key, SHA-256
checksum, size, and storage generation.
## Failure behavior
- Stage failure marks both manifests failed for that stage and records error messages/timestamps.
- Save failures are returned immediately and fail the command.
- Invalid/malformed manifest files fail load with explicit validation/decode errors.
`current/commit-pointer.json` is the sole mutable selector for the new
contract. It identifies exactly one run-scoped `runs/{run_id}/commit.json` and
binds that object by checksum, size, and generation. Readers strictly reject
unknown fields, version mismatches, pointer/commit identity mismatches, and
objects that do not match their declaration.
## Tests to inspect before changing
- `internal/manifest/manifest_test.go`
- `internal/manifest/run_manifest_test.go`
- `internal/manifest/store_test.go`
- `internal/app/runner_test.go`
- `internal/app/run_control_test.go`
- `internal/app/resume_run_stage_test.go`
The reader retains a temporary, clearly isolated compatibility path for a
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
path is removable after migration and is never used to write new state.
## Architectural invariants
- Session manifest is authoritative for stage progression across invocations.
- Run manifest is invocation-scoped and never replaces session manifest as progress authority.
- Manifest writes are atomic and deterministic (JSON + newline, temp rename pattern).
## Persistence Semantics
`manifest.LocalStore`:
- validates loaded documents;
- normalizes missing maps/stage records;
- writes through a sibling temporary file, syncing the completed file and
destination directory after atomic replacement;
- updates `updated_at` on save.
If the operating system or filesystem cannot sync a directory, save returns an
explicit error instead of claiming crash-durable replacement. A returned error
after the rename can therefore leave the new manifest visible but not confirmed
durable; callers must reload it before retrying.
## Execution Semantics
The application runner marks an executing stage running and then succeeded or
failed in both manifests, persisting each transition. On success it records
outputs, logs, generated configuration references, and metadata. Artifact
records may include optional contract and external provenance objects; old
manifests remain compatible when those fields are absent. A successful forced
rerun marks only succeeded downstream session-stage records stale.
Starting an execution clears the current session-stage record's prior outputs,
logs, generated configuration references, and metadata. Failed and skipped
transitions enforce the same clearing rule directly, while success repopulates
only fields returned by the new result. Marking a record stale does not clear
those details because resume validation and diagnosis may still require them
before execution begins. Invocation run manifests remain immutable audit
records of their own outcomes.
A stage may explicitly return a skipped disposition and stable reason. The
runner persists that outcome in both manifests, clears older outputs for the
session-stage record along with older logs, generated configuration references,
and metadata, then applies any bounded details from the current skip and
continues. This self-skip is distinct from deciding not to execute an
already-succeeded stage and is reconsidered on later runs. Skipped results
cannot contain outputs.
When an already-succeeded stage is skipped, the invocation run manifest records
the `skip` action and reason. The session manifest deliberately retains its
existing succeeded record because it remains the cross-invocation progress
authority. Stages with a resume validator, currently extraction, may reject an
otherwise eligible skip when the recorded durable result is obsolete; the
runner marks it stale and executes it.
Session manifest is the authoritative stage-progress ledger across invocations.
Run manifest is invocation-scoped audit state.
After a publish commits remotely, any configured local cleanup is first recorded
as a session-manifest obligation before deletion begins. Each target becomes
complete only after its confined deletion (or safe absence check) and a
successful manifest save. An incomplete obligation is retried on later
invocations independently of their selected stages and retains the committed
run and remote identity that authorized it.
Each invocation derives campaign, session, run, local-path, and remote-prefix
metadata from the validated resolved configuration as one projection. A persisted
session manifest must agree on campaign and session identity before execution;
the current projection is refreshed for every invocation while stage progress,
inputs, and durable artifacts remain session history.
For handled failures after an invocation record is created, the runner records
the failure on the session ledger and persists it before persisting the failed
run audit record. This preserves the resume authority while making a partial
persistence disagreement visible. Abrupt process death remains an accepted case
where a durable running record can require operator interpretation.
## Invariants
- stage resume/skip decisions are session-manifest driven.
- running, failed, and self-skipped stages do not retain result payloads from
an earlier success.
- stale stages retain prior details until replacement execution starts.
- force reruns stale downstream succeeded stages.
- run manifest does not replace session manifest as progress authority.
- remote commitment is established by a verified current pointer and remote
commit relationship, never by a mutable session-manifest boolean.
## Implementation And Tests
- Models and transitions: `internal/manifest/manifest.go`,
`internal/manifest/run_manifest.go`
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
`internal/artifacts/current_state_commit.go`,
`internal/artifacts/current_state_legacy.go`
- Persistence and validation: `internal/manifest/store.go`
- Package tests: `internal/manifest/*_test.go`
- Assembled execution behavior: `internal/app/runner_test.go`,
`internal/app/run_stage_test.go`

98
docs/internal/overview.md Normal file
View File

@@ -0,0 +1,98 @@
# Internal Overview
This document is the implemented component map for Narratio. Normative system
boundaries and dependency direction belong in
[Architecture](../policy/architecture.md). User and operator contracts belong
in the [CLI](../cli.md), [Configuration](../config.md),
[Operations](../operations.md), and [Troubleshooting](../troubleshooting.md).
Externally observable tool and format contracts belong under
[Integrations](../integrations/).
## Execution Path
```text
cmd/narratio -> internal/app -> configuration and production composition
-> internal/stage -> adapters and external systems
-> manifests and artifact resolution -> durable local/remote output
```
The executable delegates process behavior to the application boundary. The
application resolves configuration, composes concrete collaborators, acquires
session safety controls, and runs commands. Pipeline commands execute the
canonical stage sequence through adapter interfaces, while manifests record
progress and artifact services resolve durable inputs and outputs.
## Components
| Area | Implemented owners | Responsibility |
| --- | --- | --- |
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
| Configuration | `internal/config` | Strict YAML loading, discovery, defaults, normalization, session templating, and validation. |
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
The application boundary composes concrete implementations. Stages depend on
Narratio-level contracts; external transport and SDK details remain in
adapters. The normative rules for these relationships remain in
[Architecture](../policy/architecture.md).
## Pipeline Stage Set
The implemented canonical order is:
1. [`prepare`](stage-prepare.md)
2. [`transcribe`](stage-transcribe.md)
3. [`merge`](stage-merge.md)
4. [`polish`](stage-polish.md)
5. [`normalize`](stage-normalize.md)
6. [`trim`](stage-trim.md)
7. [`extract`](stage-extract.md)
8. [`render`](stage-render.md)
9. [`analyze`](stage-analyze.md)
10. [`publish`](stage-publish.md)
11. `notify` (no-op)
`notify` currently has no persisted pipeline outputs and uses the explicit
`noop` notification mode. The focused stage documents own implementation
mechanics. The
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
and execution semantics.
## Focused Documentation
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
failure behavior, and test surfaces.
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
resolution, previous requirements, and current-state helpers.
- [Manifest Internals](manifest.md): session and run records, persistence, and
execution transitions.
- [Storage Internals](storage.md): object-store interface and S3 behavior.
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
guardrails.
- [Restore Internals](command-restore.md): discovery, planning, execution, and
reporting.
- [`prepare`](stage-prepare.md)
- [`transcribe`](stage-transcribe.md)
- [`merge`](stage-merge.md)
- [`polish`](stage-polish.md)
- [`normalize`](stage-normalize.md)
- [`trim`](stage-trim.md)
- [`extract`](stage-extract.md)
- [`render`](stage-render.md)
- [`analyze`](stage-analyze.md)
- [`publish`](stage-publish.md)
Use this map to find an owner, then read its focused documentation and tests
before changing behavior.
The stage registry is implemented in `internal/stage/placeholders.go` and its
ordering is protected by `internal/app/planner_test.go`. Cross-invocation skip,
force, failure, and invalidation behavior is exercised in
`internal/app/runner_test.go` and `internal/app/run_stage_test.go`.

View File

@@ -1,80 +1,73 @@
# Stage: analyze
## Purpose
Execute selected configured Scriptorium artifacts in deterministic dependency order and materialize successful outputs to canonical session artifact paths.
## Inputs and outputs
Inputs:
- configured artifact definitions from `pipeline.scriptorium.artifacts`;
- selected artifact filter (`--artifacts`) when provided;
- resolved artifact sources from resolver/catalog.
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
Source types used by analyze:
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`;
- configured artifacts: `narratio.artifact.<artifact_key>`;
- canonical previous-session artifacts: `narratio.previous_session.artifact.<artifact_key>`.
## Inputs
Outputs:
- materialized configured artifact files at each configured `output_path`;
- stage metadata (`generated_artifacts`, `reused_artifacts`, selected/order info).
- configured artifacts from `pipeline.scriptorium.artifacts`
- optional selected artifact keys supplied through the stage environment
- built-in, configured, extraction, and previous-session source references in
artifact inputs
## Boundaries
Owns:
- runtime artifact catalog construction;
- selected-artifact planning and dependency ordering;
- per-input resolution and required/optional handling;
- Scriptorium render/run invocation;
- run-local output generation and canonical materialization.
Supported source families:
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
`narratio.input.glossary`
- configured artifacts: `narratio.artifact.<key>`
- extraction lanes: `narratio.extraction.<key>`
- previous-session cache: `narratio.previous_session.artifact.<key>`
Does not own:
- prepare-time previous-session hydration;
- object-store access for previous-session sources;
- publish output rule behavior.
## Outputs
## Config fields used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.scriptorium.binary`
- `pipeline.scriptorium.config_path`
- `pipeline.scriptorium.timeout`
- `pipeline.scriptorium.render_debug`
- `pipeline.scriptorium.artifacts.<name>.*`
- one materialized output per executed configured artifact (`output_path`)
- stage metadata describing selected/generated/reused artifacts
## External adapters used
- Scriptorium adapter:
- optional `RenderArtifact` when render-debug is enabled;
- `RunArtifact` for artifact generation.
## Key Behavior
## State and manifest behavior
- If Scriptorium config is absent, or no artifacts are executable after filtering, analyze returns success metadata with `skipped=true`.
- Builds runtime catalog with built-ins and configured `narratio.artifact.<name>` entries.
- Non-executable configured artifacts may still be marked available from existing canonical output files.
- Resolves canonical previous-session sources from local prepared `previous/` cache:
- prefers manifest-backed previous input paths when present;
- may fall back to current-session `previous/` filesystem paths.
- Analyze does not call object storage for canonical previous-session source resolution.
- Required canonical previous-session input missing:
- fails with guidance to run `narratio run-stage --force prepare`.
- Optional missing sources are omitted from adapter input paths.
- when Scriptorium is absent or no configured artifact is executable, completes
successfully with no outputs and records explanatory metadata. This is not an
explicit self-skip: both manifests record success, satisfy publish's
prerequisite, and an ordinary later run reuses the result until forced.
- builds a runtime artifact catalog containing built-ins, configured artifacts,
and configured extraction lanes. Extraction availability is hydrated only
from compatible successful extraction evidence.
- uses enabled configured artifacts by default. An explicit `--artifacts`
selection is a one-invocation override: it makes exactly the named configured
artifacts executable even when disabled, and does not automatically include
dependencies. A selected artifact's dependencies must instead already be
available to the catalog.
- marks non-executable configured artifacts as reusable when output files already exist.
- validates selected artifact dependency order (cycle-safe topo ordering).
- resolves required/optional inputs per artifact source definition.
- omits an unavailable optional input; an unavailable required input fails.
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
- resolves previous-session sources from local `previous/` cache only.
- runs optional render-debug, then artifact execution.
- validates non-empty output files and materializes canonical outputs.
## Skip and resume behavior
- Runner-level skip applies when analyze is already `succeeded` and `--force` is not set.
- Analyze is stage-scoped for resume; no per-artifact manifest resume state.
- `--artifacts` filters executable artifacts but does not imply force rerun.
## Failure Semantics
## Failure behavior
- Fails on dependency-order violations, missing required inputs, resolver validation failures, adapter errors, and missing/empty generated outputs.
- Required unavailable configured artifact source (`narratio.artifact.<name>`) fails before invocation.
- Required canonical previous-session source fails with prepare-rerun guidance.
- required missing configured/previous-session inputs fail.
- missing required prepared stable input source includes prepare rerun guidance.
- missing required previous-session source includes prepare rerun guidance.
- missing required `narratio.transcript.final_markdown` or
`narratio.transcript.final_trimmed_markdown` inputs includes render rerun
guidance.
- dependency cycles or unavailable required dependencies fail.
- adapter validation failures fail stage.
## Tests to inspect before changing
- `internal/stage/analyze_test.go`
- `internal/artifacts/catalog_test.go`
- `internal/artifacts/artifact_resolver_test.go`
- `internal/app/restore_workflow_test.go`
## Invariants
## Architectural invariants
- Canonical previous-session behavior is local-cache only during analyze.
- Generated outputs are validated and materialized before stage success is recorded.
- Resolver/catalog decisions stay deterministic and validation-gated.
- `analyze` performs no remote storage calls for previous-session source resolution.
- output provenance and metadata are deterministic per execution.
## Related Contracts And Tests
- [Configuration](../config.md#scriptorium-artifact-entries) owns artifact
fields and source-selection rules.
- [CLI](../cli.md) owns user-visible artifact selection.
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
- Implementation and tests: `internal/stage/analyze.go`,
`internal/stage/analyze_test.go`

View File

@@ -0,0 +1,90 @@
# Internal: Extract Stage
## Responsibility
`extract` runs after `trim` and before `render`. It converts the canonical
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
An omitted or disabled Notarius section makes the stage explicitly self-skip
with reason `notarius_disabled`, no outputs, and no Notarius runner.
The external protocol is documented in the
[Notarius integration contract](../integrations/notarius.md). Configuration
fields belong in [Configuration](../config.md), and physical paths and force
procedures belong in [Operations](../operations.md).
## Lifecycle
`internal/stage/extract.go`:
1. resolves the final trimmed transcript from the shared artifact catalog;
2. resolves and fingerprints the Notarius invocation contract;
3. creates a run-local staging directory and invokes the injected
`notarius.Runner`;
4. validates the successful receipt, confined index, configured required lane
descriptors, and regular payload files;
5. atomically promotes the complete bundle to its immutable durable location;
6. records one non-selectable `notarius_index` output and one selectable
`notarius_lane` output per configured lane; and
7. registers each lane as `narratio.extraction.<output_key>` for downstream
Scriptorium and publish resolution.
Lane records retain checksum, contract, producer run ID, and Notarius system,
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
root, receipt, diagnostic paths, rejection/warning summaries, producing
Narratio run ID, the resolved trimmed-input identity, and invocation
fingerprint. The input identity binds the exact transcript bytes, canonical
source ID, producer stage/output/run identity, and resolution provenance.
Validation completes before
promotion, so a rejected result cannot expose a partial durable bundle.
Any executed extraction outcome that replaces a different effective outcome
marks succeeded downstream stages stale. Repeating the same disabled self-skip
with no outputs is stable and does not repeatedly invalidate downstream stages.
## Resume Validation
`internal/stage/extract_resume.go` permits a skip only when the existing stage
record succeeded and still matches the current invocation fingerprint. The
fingerprint covers the resolved executable and config paths, pipeline ID,
timeout, working directory, sorted configured output contracts, and the current
direct trimmed-transcript identity. The same identity is resolved again for
artifact evidence, so changing the current transcript bytes or producer
identity makes the prior extraction obsolete.
The validator then checks the producing run identity, canonical immutable
bundle root, path confinement and absence of symlink components, receipt
identity, exactly one canonical index, the exact configured source set,
contracts and provenance, regular-file status, and stored checksums. Missing or
obsolete results are non-resumable and run again; unsafe filesystem conditions
return an error rather than silently accepting or replacing data.
The fingerprint cannot observe files imported by Notarius configuration,
profile contents, prompt/module definitions, or other transitive inputs.
Operators must force extraction after changing any such input.
## Failure Behavior
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
index compatibility, required-lane rejection, payload inspection, checksum, or
promotion errors fail the stage through ordinary manifest transition handling.
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
not given selectable extraction sources unless the complete configured result
has passed validation and promotion.
When a replacement attempt begins, the current session-stage record no longer
advertises payload from the previous success. A failed replacement therefore
has no current outputs, logs, generated configuration references, or metadata,
while the earlier invocation manifest and immutable promoted bundle remain
available for audit and recovery.
## Implementation And Focused Tests
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
`internal/stage/extract_resume.go`,
`internal/stage/extract_test.go`
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
`internal/adapters/notarius/subprocess_test.go`
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
`internal/artifacts/extraction_catalog_test.go`
- Composition and downstream behavior: `internal/app/runner_test.go`,
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`

View File

@@ -1,63 +1,37 @@
# Stage: merge
## Purpose
Normalize per-speaker raw transcripts and merge them into the base transcript via Seriatim.
## Inputs and Outputs
Inputs:
Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Inputs
- `transcripts/raw/*.json`
- `inputs/speakers.yml`
- `inputs/autocorrect.yml`
Outputs:
## Outputs
- `transcripts/base.json`
- optional `artifacts/seriatim.report.json` (when report enabled)
- optional `artifacts/seriatim.report.json`
## Boundaries
Owns:
- Raw transcript discovery/validation
- Per-input normalize calls to Seriatim
- Final merge call to Seriatim
- Run-local log/config/report path wiring
- Materialization of base/report outputs to canonical paths
## Key Behavior
Does not own:
- Transcript polishing or downstream artifact generation
- discovers and validates raw transcript inputs.
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
- merges normalized inputs (`seriatim.Run`) into base transcript.
- validates merged transcript and optional report JSON.
- materializes canonical outputs and records stage logs/generated configs.
## Config Fields Used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.seriatim.binary`
- `pipeline.seriatim.timeout`
- `pipeline.seriatim.output_schema`
- `pipeline.seriatim.coalesce_gap`
- `pipeline.seriatim.report`
- `pipeline.seriatim.env.*`
## Invariants
## External Adapters Used
- Seriatim adapter:
- `Normalize` for each raw input
- `Run` for final merge
- merge always consumes normalized forms of raw inputs.
- base transcript must validate before stage success.
- report output is config-gated.
## State and Manifest Behavior
- Reads transcript inputs from transcribe stage outputs in manifest when present; falls back to canonical raw directory.
- Writes run-local outputs/logs/config under `runs/{run_id}/merge/...` when enabled.
- Materializes canonical base transcript and optional report.
- Records normalized-input provenance and adapter metadata in stage metadata.
## Related Contracts And Tests
## Skip and Resume Behavior
- Runner-level skip applies when already succeeded and not forced.
- Forced rerun of this or upstream stages can stale downstream succeeded stages via runner invalidation.
## Failure Behavior
- Fails on missing/invalid raw transcripts, missing speakers/autocorrect files, normalize failure, merge failure, invalid base output JSON, or invalid report JSON when enabled.
## Tests to Inspect Before Changing
- `internal/stage/merge_test.go`
- `internal/adapters/seriatim/subprocess_test.go`
## Architectural Invariants
- Merge consumes normalized forms of each raw transcript.
- Base transcript must validate before materialization.
- Report output is optional and gated by config.
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
- [Configuration](../config.md#pipeline) owns operator-selected Seriatim values.
- Implementation and tests: `internal/stage/merge.go`,
`internal/stage/merge_test.go`

View File

@@ -1,56 +1,34 @@
# Stage: normalize
## Purpose
Normalize the polished transcript into the full final transcript and optionally emit a normalize report.
## Inputs and Outputs
Inputs:
Normalize polished transcript into final transcript using Seriatim.
## Inputs
- `transcripts/polished.json`
Outputs:
## Outputs
- `transcripts/final.json` (or configured normalize output path)
- optional `artifacts/seriatim.normalize.report.json`
## Boundaries
Owns:
- Polished transcript discovery/validation
- Normalize request construction and invocation
- Optional normalize report wiring
- Promotion of final transcript and optional report
## Key Behavior
Does not own:
- Bounds detection or segment trimming
- resolves polished transcript from manifest outputs/canonical fallback.
- applies `pipeline.normalize` config or default normalize config.
- runs Seriatim normalize with configured timeout/binary.
- validates normalized transcript and optional report.
- materializes canonical outputs and records logs/generated configs.
## Config Fields Used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.normalize.output_path`
- `pipeline.normalize.output_schema`
- `pipeline.normalize.report`
- `pipeline.seriatim.binary`
- `pipeline.seriatim.timeout`
## Invariants
## External Adapters Used
- Seriatim adapter (`Normalize`).
- final transcript must validate as processed transcript JSON (`segments` array).
- normalize defaults are applied when `pipeline.normalize` is unset.
## State and Manifest Behavior
- Reads polished transcript from polish outputs in manifest when present; falls back to canonical path.
- Uses run-local output/report/log/config paths when run layout is enabled.
- Promotes canonical final transcript and optional normalize report.
- Records adapter/result metadata including source path selection.
## Related Contracts And Tests
## Skip and Resume Behavior
- Runner-level skip applies when already succeeded and not forced.
- Forced reruns can stale downstream succeeded stages.
## Failure Behavior
- Fails on missing/invalid polished transcript, adapter error, invalid final output, or invalid report output when report enabled.
## Tests to Inspect Before Changing
- `internal/stage/normalize_test.go`
- `internal/adapters/seriatim/subprocess_test.go`
## Architectural Invariants
- Final output must validate as transcript-compatible JSON (`segments` array required).
- Default normalize config is applied when `pipeline.normalize` is unset.
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
- [Configuration](../config.md#pipeline) owns normalize fields and defaults.
- Implementation and tests: `internal/stage/normalize.go`,
`internal/stage/normalize_test.go`

View File

@@ -1,69 +1,37 @@
# Stage: polish
## Purpose
Polish the base transcript with Audita and produce a polished transcript for downstream normalization/analyze.
## Inputs and Outputs
Inputs:
Run Audita polishing on base transcript and produce polished transcript.
## Inputs
- `transcripts/base.json`
- `inputs/glossary.yml`
Outputs:
## Outputs
- `transcripts/polished.json`
- optional `artifacts/audita.report.json` (when report enabled)
- optional `artifacts/audita.report.json`
## Boundaries
Owns:
- Base transcript discovery/validation
- Audita invocation request construction
- Run-local logs/config/work-dir/report wiring
- Promotion of polished transcript and optional report
## Key Behavior
Does not own:
- Upstream merge normalization
- Downstream normalize/trim/analyze logic
- resolves base transcript from merge outputs/canonical fallback.
- invokes an Audita runner configured with static model/runtime options; the
invocation supplies paths and modules.
- validates processed transcript structure (`segments` array required).
- validates optional report JSON.
- materializes canonical outputs; records logs/generated config and adapter metadata.
## Config Fields Used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.audita.binary`
- `pipeline.audita.timeout`
- `pipeline.audita.llm_api_key_env`
- `pipeline.audita.modules`
- `pipeline.audita.base_url`
- `pipeline.audita.model`
- `pipeline.audita.transcript_description`
- `pipeline.audita.config_path`
- `pipeline.audita.output_schema`
- `pipeline.audita.work_dir_retention`
- `pipeline.audita.total_llm_concurrency`
- `pipeline.audita.proposal_llm_concurrency`
- `pipeline.audita.validation_model`
- `pipeline.audita.validation_llm_concurrency`
- `pipeline.audita.report`
## Invariants
## External Adapters Used
- Audita adapter (`env.Audita.Run`).
- polished transcript schema validation is mandatory.
- report output is config-gated.
## State and Manifest Behavior
- Reads base transcript from merge manifest outputs when available; falls back to canonical base path.
- Uses run-local output/report/log/config/scratch paths when run layout is enabled.
- Promotes canonical `transcripts/polished.json` and optional report.
- Records adapter invocation metadata, credential presence signal, and output provenance in stage metadata.
## Related Contracts And Tests
## Skip and Resume Behavior
- Runner-level skip applies when already succeeded and not forced.
- Forced rerun can stale downstream succeeded stages via runner invalidation.
## Failure Behavior
- Fails on missing/invalid base transcript, missing glossary, adapter error, invalid polished output shape (`segments` array required), or invalid report JSON when enabled.
## Tests to Inspect Before Changing
- `internal/stage/polish_test.go`
- `internal/adapters/audita/subprocess_test.go`
## Architectural Invariants
- Polished transcript must contain a top-level `segments` array.
- Report behavior is strictly config-gated.
- Stage output canonicalization always ends at `transcripts/polished.json`.
- [Audita](../integrations/audita.md) owns subprocess, validation, and failure
semantics.
- [Configuration](../config.md#pipeline) owns operator-selected Audita values.
- Implementation and tests: `internal/stage/polish.go`,
`internal/stage/polish_test.go`

View File

@@ -1,124 +1,65 @@
# Stage: prepare
## Purpose
Materialize canonical current-session input state and provenance before downstream stages run.
Prepare owns:
- local input file materialization (`inputs/**`);
- audio input materialization (`audio/**`);
- previous-session cache hydration (`previous/**`) for canonical previous-session artifact sources.
Materialize canonical current-session inputs before processing stages.
## Inputs and outputs
Inputs:
- resolved config/campaign/session (`pipeline.yml`, `campaign.yml`, `session.yml`);
- remote session provenance when `session.yml` was loaded from S3;
- campaign or session input files (`speakers`, `autocorrect`, `glossary`);
- audio source:
- local: `session.inputs.audio_dir` or `session.inputs.audio_files`;
- S3: `session.inputs.audio_s3.prefix`;
- configured enabled Scriptorium artifact inputs (for previous-session requirement scanning);
- remote previous-session current publish state when previous hydration is required.
## Inputs
Outputs:
- `inputs/campaign.yml`;
- `inputs/session.yml`;
- `inputs/pipeline.resolved.yml`;
- `inputs/speakers.yml`;
- `inputs/autocorrect.yml`;
- `inputs/glossary.yml`;
- `audio/*.flac` in canonical session `audio/`;
- optional `previous/manifest.json`;
- optional `previous/artifacts/**`;
- deterministic `manifest.Inputs` records with checksums and provenance metadata.
- resolved campaign, session, and pipeline configuration
- stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
- one resolved local or S3 audio source
- enabled configured artifact input requirements for previous-session sources
## Boundaries
Owns:
- input path resolution and materialization;
- S3 audio list/download/copy flow;
- previous-session artifact requirement collection from enabled configured artifacts;
- previous cache lifecycle when requirements exist (clear and rehydrate managed `previous/` state).
## Outputs
Does not own:
- transcript or artifact generation;
- analyze-stage source resolution;
- publish commit behavior.
- `inputs/campaign.yml`
- `inputs/session.yml`
- `inputs/pipeline.resolved.yml`
- `inputs/speakers.yml`
- `inputs/autocorrect.yml`
- `inputs/glossary.yml`
- `inputs/players.yml`
- `inputs/party.yml`
- `audio/*.flac`
- optional `previous/manifest.json`
- optional `previous/artifacts/**`
- deterministic `manifest.inputs` entries (checksums + provenance)
## Config fields used
- `session.session_id`
- `session.previous_session_id`
- `session.campaign`
- `session.inputs.speakers_file`
- `session.inputs.autocorrect_file`
- `session.inputs.glossary_file`
- `session.inputs.audio_dir`
- `session.inputs.audio_files`
- `session.inputs.audio_s3.prefix`
- `pipeline.workspace.root`
- `pipeline.spool.root`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix`
- `pipeline.scriptorium.artifacts.<name>.enabled`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.source`
- `pipeline.scriptorium.artifacts.<name>.inputs.<key>.required`
- `campaign.campaign_id`
- `campaign.inputs.speakers_file`
- `campaign.inputs.autocorrect_file`
- `campaign.inputs.glossary_file`
## Key Behavior
## External adapters used
- `storage.ObjectStore` for:
- S3 audio listing/downloads;
- previous-session current pointer/manifest/artifact object checks and downloads.
- validates required config/store state.
- enforces local audio vs S3 audio mutual exclusivity.
- rejects duplicate explicit local audio sources after resolution.
- gives distinct local source paths with the same basename deterministic unique
prepared filenames so neither source is overwritten.
- materializes S3 audio through spool/cache-aware logic.
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
- clears managed `previous/` state on every invocation, then, when requirements exist:
- resolves the pointer-selected previous source through the shared resolver;
- downloads previous manifest/artifacts;
- records previous inputs in `manifest.inputs`.
## State and manifest behavior
- Ensures workspace layout exists.
- Materializes canonical input files and audio files.
- For S3 audio, uses run-scoped spool for active downloads and durable cache for reusable audio files; cache hits copy directly to work audio without downloading the object again.
- Records `inputs/session.yml` provenance as local `session_config` or remote `session_config.s3`.
- Resolves campaign-provided stable input paths relative to `campaign.yml`.
- Resolves session-provided stable input overrides relative to `session.yml`.
- Scans enabled configured artifact inputs for canonical sources:
- `narratio.previous_session.artifact.<artifact_key>`
- If one or more canonical previous-session requirements exist:
- clears managed `previous/` state;
- hydrates required/optional previous artifacts from the configured previous sessions committed publish current state;
- writes `previous/manifest.json` and hydrated `previous/artifacts/**`;
- stores publish-relative artifact paths such as `artifacts/session_recap.md` as `previous/artifacts/session_recap.md`, not `previous/artifacts/artifacts/session_recap.md`;
- records hydrated previous inputs in `manifest.Inputs` with source `previous_session_publish.current`.
- If no canonical previous-session requirements exist, prepare does not manage `previous/`.
- `manifest.Inputs` is sorted deterministically by `(kind, path)`.
- S3 audio `manifest.Inputs` retain S3 provenance and include `cache_path`; `spool_path` is present only when the current prepare invocation downloaded the file.
Required previous-session inputs fail when unavailable; optional missing inputs
are typed skipped results. Committed sources use their exact source-to-destination
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
## Required and optional previous-session behavior
- `previous_session_id` unset:
- if any referenced previous artifact is required: fail;
- if all referenced previous artifacts are optional: continue and omit them.
- Previous session publish current pointer or manifest missing:
- if any referenced previous artifact is required: fail;
- if all referenced previous artifacts are optional: continue and omit missing ones.
- Missing required previous artifact object: fail.
- Missing optional previous artifact object: omit.
- Downloaded previous artifacts must validate as non-empty files.
## Invariants
## Skip and resume behavior
- Runner-level skip remains authoritative:
- if `prepare` already succeeded and run is not forced, `prepare` does not run and no hydration/download occurs.
- If `prepare` runs (including with `--force`), it owns managed `previous/` state for canonical previous-session inputs.
- only `prepare` hydrates canonical `previous/` cache state.
- managed previous artifacts are stored under `previous/artifacts/**` without
duplicate `artifacts/artifacts/` nesting.
- managed `previous/` state represents only the current requirement set.
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
## Failure behavior
- Fails on missing required input files, invalid audio-source combinations, empty/duplicate audio inputs, missing object store for S3 modes, and remote access/download/validation errors.
- For required canonical previous-session inputs, analyze-time missing-input guidance is to rerun:
- `narratio run-stage --force prepare`
## Related Contracts And Tests
## Tests to inspect before changing
- `internal/stage/prepare_test.go`
- `internal/stage/prepare_previous_test.go`
- `internal/artifacts/previous_requirements_test.go`
- `internal/app/runner_test.go`
## Architectural invariants
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive.
- Storage keys are computed by callers using path helpers; storage adapter receives explicit keys.
- `prepare` is the only stage that hydrates canonical previous-session cache state.
- [Configuration](../config.md) owns audio selection, stable input fields, and
previous-session settings.
- [Operations](../operations.md) owns physical input, audio, spool, cache, and
previous-state layout.
- [Storage Internals](storage.md) and [Artifact Internals](artifacts.md) explain
the internal collaborators.
- Implementation and tests: `internal/stage/prepare.go`,
`internal/stage/prepare_test.go`, `internal/audio/s3_audio_test.go`,
`internal/previouscache/*_test.go`

View File

@@ -1,88 +1,99 @@
# Stage: publish
## Purpose
Publish durable run/session state to object storage, then atomically advance remote current state.
## Inputs and Outputs
Inputs:
- session manifest and prerequisite stage records
- run root contents under `runs/{run_id}/`
- publish output rules with artifact `source` IDs and publish `dest` paths (`pipeline.publish.outputs`)
- effective source-based publish locks from static config and remote session lock store
- session-level `previous/**` cache files when present
Upload run/session outputs to object storage and atomically advance remote current state.
Outputs:
- uploaded run files under `{session_prefix}/runs/{run_id}/...`
- uploaded published outputs under `{session_prefix}/...`
- uploaded session previous-cache files under `{session_prefix}/previous/...` when present
- `{session_prefix}/current/manifest.json`
- `{session_prefix}/current/run_id.txt` written last
## Inputs
## Boundaries
Owns:
- publish enable/disable gate behavior
- prerequisite stage success enforcement
- run file collection and upload (excluding `audio/`)
- publish output rule resolution and upload
- publish lock enforcement
- session previous-cache file collection/upload
- commit pointer publish order
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
- invocation-scoped run files
- resolved publish output rules
- effective publish locks (static + remote merged lock set), revalidated at the
remote commit boundary
- durable previous-session cache files when present
Does not own:
- stage execution before publish
- post-publish local cleanup policy execution (handled by app cleanup logic)
## Outputs
## Config Fields Used
- `pipeline.publish.enabled`
- `pipeline.publish.upload_run`
- `pipeline.publish.outputs`
- `pipeline.publish.locks`
- `{session_prefix}/locks.yml` loaded by app orchestration before publish execution
- `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.root_prefix`
- `pipeline.workspace.root`
- `session.campaign`
- `session.session_id`
- uploaded invocation record and selected publish outputs;
- uploaded durable previous-session cache files when present;
- immutable run-scoped commit manifest; and
- current commit pointer, written last.
## External Adapters Used
- Object storage backend (`env.ObjectStore`) for upload/list primitives.
Exact remote placement and the operator workflow belong in
[Operations](../operations.md#publish-workflow).
## State and Manifest Behavior
- Requires `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` status `succeeded`.
- Resolves bucket/prefix from manifest identity first, then config fallback.
- Uploads session `previous/**` files as durable session state when the local `previous/` directory exists.
- Skips top-level published output uploads for effective locked sources; run-local materialized outputs remain unchanged.
- When selected configured artifact keys are supplied, skips publish rules for unselected `narratio.artifact.<key>` sources; built-in transcript and bounds outputs still publish.
- Effective locks are the union of `pipeline.publish.locks` and remote `{session_prefix}/locks.yml`; static pipeline locks win on duplicate sources.
- Writes metadata including:
- upload counts/paths
- `previous_files_uploaded` and `previous_uploaded_paths`
- `published_files_uploaded` and `published_paths`
- `skipped_optional_outputs`
- `skipped_unselected_outputs`
- `locked_output_count` and `locked_outputs`
- `current_manifest_key`
- `current_run_id_key`
- `current_pointer_written`
- On skipped publish path, returns metadata with `skipped=true` and pointer not written.
## Key Behavior
## Skip and Resume Behavior
- Stage may self-skip (metadata skip) when publish disabled or run upload disabled.
- Runner-level skip also applies for previously succeeded stage unless forced.
- when publishing or run upload is disabled, completes successfully with no
outputs and records explanatory metadata. This is not an explicit self-skip:
both manifests record success, and an ordinary later run reuses that result
until publish is forced.
- validates prerequisite stage success and object-store availability.
- derives a deterministic run-archive allowlist from the validated run
`manifest.json`: declared run-local outputs, logs, generated configs, and the
manifest itself. Unlisted workspace files are not archive candidates.
- opens each archive candidate beneath its archive root without following
symlinked ancestors or leaf entries, verifies that it is a regular file and
checks a declared checksum when present, then streams the opened descriptor.
- derives the durable previous-cache archive from its validated manifest using
the same confinement and regular-file checks.
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
- publishes extraction lanes only through explicit configured output rules;
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
- selected artifact filter applies to configured artifact sources only.
- locked outputs are skipped intentionally (including required ones).
- optional missing outputs are skipped; required missing unlocked outputs fail.
- creates one complete immutable source-to-destination mapping before upload;
- uploads and verifies every declared immutable object and the commit manifest;
- updates `current/commit-pointer.json` exactly once, last; and
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
- rechecks remote lock state immediately before the pointer update. A newly
committed lock aborts selection, leaving any uploaded immutable attempt
unselected.
- reads the mutable remote lock document through a direct limit-plus-one read
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
with the opened body for conditional updates. Oversized lock documents fail
before YAML decoding; published artifact payloads do not use this limit.
## Failure Behavior
- Fails on missing prerequisite success, missing object store when required, missing run root, missing unlocked required output source, upload failures, or pointer write failures.
- Locked required outputs are intentional skips and do not fail publish.
- Pointer semantics are fail-safe: `current/run_id.txt` is not written if prior required uploads fail.
## Metadata Signals
## Tests to Inspect Before Changing
- `internal/stage/archive_test.go`
- `internal/app/post_archive_cleanup_test.go`
Includes counts/lists for:
- run uploads
- published output uploads
- previous uploads
- skipped optional outputs
- skipped unselected outputs
- locked outputs
- remote commit and current-pointer key paths
- the run identifier selected by the commit
## Architectural Invariants
- Run upload excludes `audio/` subtree.
- Session `previous/**` is publishable durable input/provenance state, not run-local output.
- Ordinary `--force` does not override publish locks.
- Malformed or unreadable remote lock store fails publish-capable execution before output uploads.
- `current/manifest.json` uploads before `current/run_id.txt`.
- `current/run_id.txt` is the remote publish commit marker.
## Invariants
- `current/commit-pointer.json` is the remote commit marker and is written last.
- run files, selected outputs, previous-cache files, and the committed session
manifest are all declared by an immutable commit under the run prefix.
- run and previous uploads contain only manifest-declared regular files opened
from verified descriptors; symlinks, special files, replacement races, and
undeclared entries are rejected or ignored before uploads begin.
- run-local diagnostics, including Notarius receipt and stderr files, are
archived only when recorded by the run manifest.
- publish locks are not overridden by `--force`; remote locks are revalidated
immediately before current-state selection.
- post-commit local cleanup is authorized by the committed publish metadata and
is durably recorded by the application lifecycle before any local deletion.
The commit boundary and cleanup gate are normative architecture invariants; see
[Architecture](../policy/architecture.md#publish-commit-boundary).
## Related Contracts And Tests
- [Configuration](../config.md#publish-configuration-summary) owns output and
static-lock fields.
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
physical remote state.
- [Artifact Internals](artifacts.md) explains source resolution and current-state
helpers.
- Implementation and tests: `internal/stage/publish.go`,
`internal/stage/publish_test.go`, `internal/app/operator_helpers_test.go`,
`internal/app/post_publish_cleanup_test.go`

View File

@@ -0,0 +1,44 @@
# Stage: render
## Purpose
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
## Inputs
- `narratio.transcript.final` (`transcripts/final.json`)
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
## Outputs
- `narratio.transcript.final_markdown` -> `transcripts/final.md`
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
## Key Behavior
- uses `pipeline.render` settings (enabled/format/title/booleans).
- resolves inputs manifest-first, then canonical fallback.
- writes run-local outputs first, then materializes canonical session outputs.
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
- when `pipeline.render.enabled=false`, completes successfully with no outputs
and records explanatory metadata. This is not an explicit self-skip: both
manifests record success, and enabling render later requires a forced run.
## Failure Semantics
- missing normalized input fails with normalize rerun guidance.
- missing trimmed input fails with trim rerun guidance.
- adapter/subprocess failure fails stage.
- empty render output files fail validation.
## Invariants
- only `format: markdown` is supported.
- render stage owns production of built-in Markdown transcript sources.
## Related Contracts And Tests
- [Seriatim](../integrations/seriatim.md) owns render subprocess behavior.
- [Configuration](../config.md#pipeline) owns render fields and defaults.
- Implementation and tests: `internal/stage/render.go`,
`internal/stage/render_test.go`

View File

@@ -1,58 +1,41 @@
# Stage: transcribe
## Purpose
Generate per-speaker raw transcripts from prepared audio using WhisperX.
## Inputs and Outputs
Inputs:
- `audio/*.flac` prepared by `prepare`
Generate raw per-speaker transcripts from prepared audio using WhisperX.
Outputs:
- `transcripts/raw/<speaker>.json` for each input audio file
## Inputs
## Boundaries
Owns:
- Discovering prepared audio inputs
- Deriving speaker ids from audio basenames
- Parallel WhisperX invocation with bounded concurrency
- Validating produced JSON and materializing run-local outputs
- `audio/*.flac` from `prepare`
Does not own:
- Transcript merge/polish/normalize/trim/analyze
## Outputs
## Config Fields Used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.whisperx.transcribe_url`
- `pipeline.whisperx.language`
- `pipeline.whisperx.timeout`
- `pipeline.whisperx.retries`
- `pipeline.whisperx.retry_delay`
- `pipeline.whisperx.concurrency`
- `transcripts/raw/<speaker>.json`
## External Adapters Used
- WhisperX adapter (`env.WhisperX.Transcribe`).
## Key Behavior
## State and Manifest Behavior
- Uses run-local output paths under `runs/{run_id}/transcribe/outputs/...` when run layout is enabled.
- Validates each generated transcript JSON before materialization.
- Materializes canonical outputs to `transcripts/raw/*.json`.
- Records per-file metadata (attempts/status/duration/output path) in stage metadata.
- discovers prepared audio from manifest inputs or canonical audio directory.
- derives the transcript identity from the prepared `.flac` filename.
- dispatches WhisperX requests through a bounded worker pool.
- validates each output as JSON.
- writes run-local outputs then materializes canonical transcript outputs only
after every planned request succeeds.
## Skip and Resume Behavior
- Runner-level skip applies for previously succeeded stage unless forced.
- On forced upstream reruns, downstream succeeded stages can be marked `stale` by runner logic.
## Invariants
## Failure Behavior
- Fails if no prepared audio exists, duplicate speaker basenames are detected, adapter output path mismatches expected path, any output JSON is invalid, or one worker fails.
- Cancels in-flight workers after first terminal error.
- prepared audio identities must be unique; prepare disambiguates distinct
source paths that share a basename.
- output path returned by adapter must match requested output path.
- an empty adapter result path means the requested path; adapters cannot select
an alternate destination.
- each successful output is validated before stage success, and cancellation or
incomplete dispatch cannot be reported as a successful result.
## Tests to Inspect Before Changing
- `internal/stage/transcribe_test.go`
- `internal/app/whisperx_wiring_test.go`
## Related Contracts And Tests
## Architectural Invariants
- Speaker identity is derived from `.flac` basename and must be unique.
- Every successful speaker output must be valid JSON before materialization.
- Canonical raw transcript set is the only supported merge input surface.
- [WhisperX](../integrations/whisperx.md) owns HTTP, retry, timeout, and
cancellation semantics.
- [Configuration](../config.md#pipeline) owns concurrency and other
operator-selected values.
- Implementation and tests: `internal/stage/transcribe.go`,
`internal/stage/transcribe_test.go`

View File

@@ -1,75 +1,43 @@
# Stage: trim
## Purpose
Optionally trim the final transcript to session bounds; always produce a durable final-trimmed transcript.
## Inputs and Outputs
Inputs:
Produce a final-trimmed transcript. By default, the stage generates bounds and
applies a bounds-driven trim.
## Inputs
- `transcripts/final.json`
Outputs:
## Outputs
- `transcripts/final.trimmed.json` (or configured trim output path)
- when trim enabled: `artifacts/session_bounds.json`
## Boundaries
Owns:
- Trim-enabled switch behavior
- Bounds generation via Scriptorium artifact run
- Bounds validation against final transcript
- Keep-selector derivation and Seriatim trim invocation
- Copy-through behavior when disabled or bounds indicate unchanged transcript
## Key Behavior
Does not own:
- Upstream normalization
- Downstream artifact analysis
When `trim.enabled=true`:
- runs Scriptorium bounds artifact generation;
- optionally runs render-debug output generation;
- validates bounds payload against transcript;
- derives keep selector;
- either copies unchanged transcript or runs Seriatim trim;
- validates trimmed transcript and materializes bounds output.
## Config Fields Used
- `session.session_id`
- `session.campaign`
- `pipeline.workspace.root`
- `pipeline.trim.enabled`
- `pipeline.trim.output_path`
- `pipeline.trim.bounds.prompt_id`
- `pipeline.trim.bounds.profile_id`
- `pipeline.trim.bounds.timeout`
- `pipeline.trim.bounds.output_path`
- `pipeline.trim.bounds.transcript_input_name`
- `pipeline.trim.bounds.render_debug`
- `pipeline.trim.bounds.render_output_path`
- `pipeline.seriatim.binary`
- `pipeline.seriatim.timeout`
- `pipeline.scriptorium.binary`
- `pipeline.scriptorium.config_path`
- `pipeline.scriptorium.timeout`
When `trim.enabled=false`:
- copies normalized transcript to trimmed output.
## External Adapters Used
- Scriptorium adapter:
- optional `RenderArtifact` for bounds debug render
- `RunArtifact` for bounds output
- Seriatim adapter:
- `Trim` when bounds indicate trimming is required
## Invariants
## State and Manifest Behavior
- Reads final transcript from normalize manifest outputs when available; falls back to canonical path.
- Uses run-local outputs/logs/reports/config/scratch paths when run layout is enabled.
- Materializes canonical final-trimmed transcript and session bounds when trim is enabled.
- Records bounds diagnostics, trim action, keep selector, and adapter metadata.
- normalized transcript is required input.
- bounds output exists only in enabled trim path.
- render-debug output is diagnostic and not a declared stage output.
## Skip and Resume Behavior
- Runner-level skip applies when already succeeded and not forced.
- Forced reruns can stale downstream succeeded stages.
- When `trim.enabled=false`, stage still succeeds by copying final to final-trimmed output.
## Related Contracts And Tests
## Failure Behavior
- Fails on missing/invalid final transcript.
- With trim enabled, fails on missing adapters/config, bounds generation/validation errors, invalid bounds JSON, invalid range/segment ids, trim adapter failures, or invalid final-trimmed output.
## Tests to Inspect Before Changing
- `internal/stage/trim_test.go`
- `internal/adapters/scriptorium/subprocess_test.go`
- `internal/adapters/seriatim/subprocess_test.go`
## Architectural Invariants
- Trim never falls back to polished transcript; final transcript is required input.
- `session_bounds` output exists only for enabled trim path.
- Render-debug artifacts are diagnostics and not declared stage outputs.
- [Scriptorium](../integrations/scriptorium.md) owns bounds generation and
debug-render subprocess behavior.
- [Seriatim](../integrations/seriatim.md) owns transcript trimming behavior.
- [Configuration](../config.md#pipeline) owns trim fields and defaults.
- Implementation and tests: `internal/stage/trim.go`,
`internal/stage/trim_test.go`

View File

@@ -1,75 +1,64 @@
# Internal: Storage
## Purpose
Document Narratio's remote storage backend contracts and implementations under `internal/adapters/storage`.
## Inputs and outputs
Inputs:
- Resolved storage config (`pipeline.storage.*`).
- Already-loaded environment variables for configured S3 credentials.
- Bucket-relative object keys and local file paths from app/stage orchestration.
Explain the object-store interface and S3 implementation used by Narratio.
Remote key layout and lifecycle belong in [Operations](../operations.md), while
operator-selected storage fields and credential mechanisms belong in
[Configuration](../config.md).
Outputs:
- Listed/downloaded/uploaded object metadata (`ObjectInfo`).
- Existence checks and storage-layer errors.
## Primary Contract
## Boundaries
Owns:
- Remote object-store interface and implementation details.
- S3 client wiring and API calls.
- Object key normalization and upload/download/list primitives.
`storage.ObjectStore` interface:
Does not own:
- Session/run prefix semantics.
- Archive commit order semantics.
- Manifest updates.
- Filesystem secret loading from `pipeline.secrets.env_dir`.
- `List(ctx, prefix)`
- `Read(ctx, key)` returns an object body and the generation observed with it
- `Download(ctx, key, localPath)`
- `Upload(ctx, localPath, key, opts)`
- `UploadConditional(ctx, source, key, opts, condition)`
- `Exists(ctx, key)`
## Config fields used
- `pipeline.storage.backend`
- `pipeline.storage.s3.bucket`
- `pipeline.storage.s3.region`
- `pipeline.storage.s3.endpoint`
- `pipeline.storage.s3.force_path_style`
- `pipeline.storage.s3.access_key_id_env`
- `pipeline.storage.s3.secret_access_key_env`
Key invariant:
- callers pass full bucket-relative keys;
- storage implementations do not infer campaign/session/run prefixes.
## External adapters used
Storage package contracts:
- `ObjectStore` (active remote object-store boundary): `List`, `Download`, `Upload`, `Exists`.
- `Backend` (legacy compatibility boundary): currently implemented with `NoopBackend` only.
`ReadObjectBounded` is the shared mechanism for small control objects. It opens
one object version, returns the metadata observed with that body, rejects an
oversized known size before transfer, and still performs a context-aware
limit-plus-one read. It closes the body on every exit. Callers own the policy
limit and add the control-object category to errors; this helper is not used for
large artifact payloads.
Implementations:
- `S3Backend`: AWS SDK-backed `ObjectStore` implementation.
- `FakeBackend`: deterministic test `ObjectStore` and compatibility backend.
- `NoopBackend`: deterministic no-op compatibility backend for wiring/tests.
## Composition
## State and manifest behavior
- Storage implementations are stateless with respect to manifest/session lifecycle.
- Caller supplies fully-qualified bucket-relative keys.
- Storage layer does not infer campaign/session/run/root-prefix semantics.
- Caller controls publish ordering; storage layer executes individual operations in the order invoked.
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
resolved configuration. The application loads configured filesystem secrets
before calling it. The storage adapter consumes already-resolved values; it does
not own discovery, defaults, or configuration validation.
## Skip and resume behavior
- No storage-level skip/resume behavior.
- Skip/resume decisions are made by stage/app logic before storage calls occur.
## S3 Backend Behavior
## Failure behavior
- `NewObjectStoreFromConfig` fails when no remote backend is configured or required S3 config is missing.
- `S3Backend` constructor fails when required bucket is missing or AWS client setup fails.
- App command orchestration loads configured filesystem secrets before calling the object-store factory.
- CRUD operations return contextual errors (including not-found behavior via `Exists`).
- Key normalization is applied before operations (`\\` to `/`, leading slash trimmed).
- Remote session loading uses `List` to find the exact `session.yml` key and `Download` to materialize it to a local temp file.
- normalizes object keys.
- `List` paginates and returns normalized `ObjectInfo`.
- A truncated S3 listing must supply a new, non-empty continuation token;
otherwise listing fails with bucket and prefix context instead of looping.
- `Download` writes local files with parent directory creation.
- `Upload` streams local file and returns remote metadata.
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
match or absence precondition directly to the provider request and reports a
failed precondition without performing a local check-then-write replacement.
- `Exists` maps not-found responses to `false`.
## Tests to inspect before changing
- `internal/adapters/storage/factory_test.go`
- `internal/adapters/storage/s3_backend_test.go`
- `internal/adapters/storage/fake_test.go`
- `internal/adapters/storage/keys_test.go`
- `internal/adapters/storage/archive.go` + consumers in stage tests (`prepare`, `publish`)
## Invariants
## Architectural invariants
- Callers pass full bucket-relative keys.
- Storage backends must not prepend or infer narratio prefixes.
- Remote transport details remain isolated to storage adapter implementations.
- storage layer is stateless regarding manifest/stage progression.
- bounded reads never retain more than the caller's limit plus one byte and do
not replace owner-specific size policy.
- publish ordering semantics are owned by stage/app code, not storage adapters.
## Implementation And Tests
- Contract and S3 adapter: `internal/adapters/storage`
- Composition: `internal/app/object_store.go`
- Tests: `internal/adapters/storage/*_test.go`,
`internal/app/object_store_test.go`

View File

@@ -1,78 +1,94 @@
# Workspace internals
# Internal: Workspace
## Purpose
Define the local durable and run-local workspace model used by stages, manifests, resume, and publish.
## Inputs and Outputs
Inputs:
- `pipeline.workspace.root`
- `session.campaign`
- `session.session_id`
- generated `run_id`
Explain the helpers that construct local session and run paths, coordinate
single-writer access, and confine cleanup. The authoritative physical layout and
retention workflow belong in [Operations](../operations.md#local-state-layout).
Outputs:
- Session manifest at `{workspace.root}/work/{campaign}/{session_id}/manifest.json`
- Run manifest at `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`
- Canonical durable session directories and run-local stage trees
## Path Ownership
## Boundaries
Owns:
- Session-level path layout (`inputs/`, `audio/`, `transcripts/`, `artifacts/`, `reports/`, `logs/`, `config/`, `current/`, `runs/`, `previous/`)
- `previous/manifest.json` and `previous/artifacts/**` are reserved for previous-session cache state materialized by `prepare` or `restore`
- Run-local stage sandbox layout under `runs/{run_id}/{stage}/`
- Session lock acquisition/release (`.lock`)
`internal/artifacts` owns canonical session, run, spool, cache, and
previous-cache path construction. `SessionPathsFor` provides the session-scoped
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
consume those helpers instead of rebuilding relative paths.
Does not own:
- Stage business logic
- Remote publish semantics (documented in `stage-publish.md`)
- CLI argument parsing
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
cleanup roots and their descendants through no-follow directory handles before
removing them.
## Config Fields Used
- `pipeline.workspace.root`
- `pipeline.workspace.cleanup_after_publish`
- `pipeline.spool.root`
- `pipeline.spool.delete_audio_after_publish`
- `pipeline.cache.root`
- `pipeline.cache.s3_audio`
- `session.campaign`
- `session.session_id`
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
`EnsureWorkspaceDirectory` reapplies the directory mode after creation so a
restrictive umask cannot remove group access, while retaining existing ownership
and group. Credential paths are outside this contract; the platform-specific
operational requirements are in [Operations](../operations.md#workspace-permissions).
## External Adapters Used
None directly in this subsystem. Stages may use object storage adapters and then write local outputs into this layout.
## Run-Local Stage Layout
## State and Manifest Behavior
- Session state is persisted in the session manifest (`manifest.Manifest`).
- Invocation history is persisted per run in run manifests under `runs/{run_id}/manifest.json`.
- During each run, stage outputs are often written run-local first (`runs/{run_id}/{stage}/outputs/...`) and then materialized to canonical session paths after stage success.
- `manifest.Artifacts` entries record `ProducerRunID` for durable outputs.
- For S3 audio sessions, `prepare` records work/cache paths, S3 provenance, and spool path when the invocation downloaded the object.
- `previous/**` is reconstructed from configured previous-session requirements; restore uses the previous session's committed current publish state rather than treating current-session stored `previous/**` as authoritative.
- Durable cache state under `pipeline.cache.root` is not workspace state and is preserved by default by `narratio clean`.
- `narratio clean <id>` removes the session work root and session spool root.
- `narratio clean --all` removes all local session work under `workspace.root/work` and spool children under `spool.root`.
- `narratio clean --clear-cache` is the explicit opt-in for deleting matching S3 audio cache entries.
`internal/stage/run_local.go` maps stage outputs and diagnostics into an
invocation-scoped layout. Successful outputs are validated and atomically
materialized into canonical session paths before stage success. Managed
previous-session cache paths remain session-durable and are never redirected
into run-local output space.
## Skip and Resume Behavior
- Skip/resume decisions are made in `internal/app` (`run_control.go`, `resume.go`) using stage status in the session manifest.
- `--force` reruns selected stages and marks downstream previously-succeeded stages as `stale`.
- Workspace layout is idempotent (`EnsureLayoutFor`) and reused across runs.
Extraction uses run-local receipt, stderr, and output-root helpers, then
promotes the validated external bundle to the unique immutable Notarius bundle
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
copies only regular files and directories to a same-filesystem temporary
sibling. Source traversal uses confined directory handles and identity checks
so replacing an inspected root, directory, or file is rejected rather than
followed. The completed tree is atomically renamed without replacing an
existing destination. Exact physical paths belong in
[Operations](../operations.md#extraction-workflow).
## Failure Behavior
- Failures preserve manifests and run-local files for inspection.
- Lock conflicts fail fast via `ErrLockConflict`.
- Cleanup can fail post-publish; failure is recorded in publish stage metadata and returned by the run.
## Locking
## Tests to Inspect Before Changing
- `internal/artifacts/local_test.go`
- `internal/stage/run_local_test.go`
- `internal/app/run_control_test.go`
- `internal/app/resume_run_stage_test.go`
- `internal/app/post_archive_cleanup_test.go`
`artifacts.LocalStore` enforces the single-writer session lock via an
operating-system lock held on `.lock` (`ErrLockConflict` on contention). The
file retains owner metadata after release or process death; its existence is
not evidence that a lock is active. Command and restore flows wait for this
lock only while their context remains active, and report a release failure.
## Architectural Invariants
- Session root is campaign-aware: `{workspace.root}/work/{campaign}/{session_id}`.
- Run roots are always nested: `runs/{run_id}` under the session root.
- Run-local output materialization must end in canonical session paths.
- `previous/**` is session-durable state and must not be treated as run-local output scratch state.
- Automatic post-publish cleanup only targets run-scoped directories and must never delete configured root directories.
- Manual `clean` may delete session-scoped directories or the `workspace.root/work` directory, but it must preserve configured root directories and reject unsafe targets.
## Cleanup Semantics
Automatic post-publish cleanup:
- is created only after a successful publish commit with complete publish
metadata, then is persisted before any deletion;
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
key in publish metadata;
- consumes the resolved cleanup policy described in
[Configuration](../config.md);
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
ancestors or entries);
- retries any recorded incomplete target on later invocations even when no
publish work is selected. Missing targets are a successful, idempotent
cleanup result only after the completion evidence is saved.
Manual cleanup uses the same root-confined deletion mechanism. Invocation
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
[Operations](../operations.md#cleanup).
## Invariants
- campaign-aware session root is mandatory.
- manifest-driven stage state is durable across runs.
- cleanup guardrails prevent destructive root/out-of-scope deletion.
- ordinary workspace paths retain group-writable directory and file modes across
nested creation, replacement, and Notarius promotion.
## Implementation And Tests
- Path model and local store: `internal/artifacts/paths.go`,
`internal/artifacts/local.go`
- Run-local materialization: `internal/stage/run_local.go`
- Immutable bundle promotion: `internal/fileops/directory.go`
- Workspace modes: `internal/fileops/modes.go`
- Cleanup confinement: `internal/fileops/cleanup.go`,
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
- Tests: `internal/artifacts/paths_model_test.go`,
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
`internal/app/post_publish_cleanup_test.go`

View File

@@ -1,72 +1,271 @@
# Operations
# Operations Guide
This guide covers the implemented operator lifecycle for Narratio.
Operator workflow for running, recovering, and publishing Narratio sessions.
For field-level settings, see [docs/config.md](./config.md). For syntax/flags, see [docs/cli.md](./cli.md).
For command syntax, see [docs/cli.md](./cli.md). For field-level config, see [docs/config.md](./config.md).
## Normal Workflow
## Campaign and Session Selection
1. Ensure `pipeline.yml`, `campaign.yml`, and `session.yml` are available.
2. Ensure session audio is available (local `audio_dir`/`audio_files` or S3 prefix).
3. Run:
Campaign selection priority:
- `--campaign-file`
- `--campaign`
- `pipeline.campaigns.default_campaign_id`
Session source priority:
- `--session`
- local default search paths
- remote session object (S3) when local session file is not found and storage is configured
## Session Initialization
Use `session init` to generate a concrete session file for local or remote use.
Local file:
```bash
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
```
Remote session object:
```bash
narratio session init 2026-04-04 --remote --force
```
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
Campaigns must provide stable input files for speakers, autocorrect, glossary, players, and party. Session files may override those paths for one session. The `prepare` stage materializes them under `inputs/`; configured Scriptorium artifacts can reference prepared `players`, `party`, and `glossary` files with `narratio.input.players`, `narratio.input.party`, and `narratio.input.glossary`.
## Standard Session Workflow
1. Select pipeline/campaign/session config.
2. Validate session readiness:
```bash
narratio session validate 2026-04-04
```
3. (Optional) inspect stage decisions:
```bash
narratio session plan 2026-04-04
```
4. Run the pipeline:
```bash
narratio run 2026-04-04
```
4. Inspect status:
5. Check state:
```bash
narratio session status 2026-04-04
```
## Stage Execution and Continuation Behavior
Canonical stage order:
1. `prepare`
2. `transcribe`
3. `merge`
4. `polish`
5. `normalize`
6. `trim`
7. `extract`
8. `render`
9. `analyze`
10. `publish`
11. `notify`
Execution rules:
- succeeded stages are skipped unless `--force` is set;
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
- forcing an upstream stage marks succeeded downstream stages as `stale` before
the replacement runs; and
- an executed failure, changed self-skip, or success that replaces a different
effective upstream outcome also marks succeeded downstream stages stale. A
repeated self-skip with the same reason and no outputs is stable and does not
perpetually rerun downstream work.
An explicit self-skip is a durable `skipped` stage outcome that later runs
reconsider. It differs from successful no-output execution: disabled `render`
and `publish`, and absent or no-executable `analyze`, record `succeeded` with
metadata and no outputs. Ordinary later runs reuse those successful results;
force the affected stage after enabling or configuring it. Optional artifact
inputs are omitted only from the consuming artifact invocation and do not make
the stage self-skip.
Single-stage execution:
```bash
narratio run-stage normalize 2026-04-04 --force
```
## Artifact Selection
`--artifacts` can be used on `run`, `run-stage`, `analyze`, and `publish`.
Selection behavior:
- validates names against `pipeline.scriptorium.artifacts`;
- filters analyze execution to selected configured artifacts;
- filters publish rules for `narratio.artifact.<name>` sources only;
- does not suppress built-in transcript, bounds, or explicitly configured
`narratio.extraction.<name>` publish sources; and
- never partially selects Notarius lanes.
## Extraction Workflow
When Notarius is omitted or disabled, `extract` records an explicit skipped
outcome with reason `notarius_disabled` and no outputs. A later invocation
reconsiders the skipped stage, so enabling Notarius does not require force.
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
and preserves the complete validated Notarius bundle at:
- `artifacts/notarius/{narratio_run_id}/`
The directory is immutable once promoted. Configured lanes become
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
the bundle and `index.json` are retained for audit and resume validation but
are not selectable or published implicitly.
Starting a replacement clears the previous extraction payload from the current
session-stage record. If that replacement fails or self-skips, the current
record does not fall back to the earlier outputs. The earlier run manifest and
immutable bundle remain available for inspection, but downstream resolution
requires a new current successful extraction record.
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
and other operating systems, extraction fails before copying the bundle into a
temporary promotion tree because Narratio has no verified atomic no-replace
directory primitive there. This is an extraction limitation, not a broader
platform-support guarantee for every Narratio workflow.
## External Command Lifecycle
When an external command is cancelled or times out, Narratio terminates its
owned descendants as well as the command itself. Cancellation first requests
termination where the platform supports it, then force terminates after a
bounded wait. A command is not considered finished until its leader has been
reaped, and descendants that keep standard output or error open cannot keep
the invocation blocked. Other operating systems fail closed rather than launch
a command without tree ownership.
Subprocess stdout and stderr diagnostics are separately redacted and capped at
8 MiB per invocation. Narratio does not retain configured credential values in
these logs or their error tails; reaching a capture limit terminates the command
tree and reports which stream exceeded the limit.
Run-local diagnostics are:
- `runs/{run_id}/extract/notarius.receipt.json`
- `runs/{run_id}/extract/notarius.stderr.log`
- `runs/{run_id}/extract/notarius-output/` before durable promotion
The run-record upload is an allowlist derived from the validated run manifest,
not a workspace scan. Each declared source is opened without following
symlinked ancestors or the leaf, verified as a regular file, and streamed from
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
The durable bundle is never scanned for implicit publication; only lanes named
by explicit `pipeline.publish.outputs` rules are uploaded.
To intentionally replace the current extraction result, run:
```bash
narratio run-stage extract 2026-04-04 --force
```
Narratio automatically reruns extraction when its recorded invocation contract
or durable output validation changes. It cannot fingerprint configuration
files, profiles, prompts, modules, or references loaded transitively by
Notarius. Force extraction after changing any of those inputs, even when the
top-level Narratio and Notarius config paths remain the same. A forced extract
marks successful downstream stages stale. Ordinary extraction failures or
outcome changes also stale affected downstream stages, while an identical
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
## Publish Workflow
Publish is the stage that commits remote current state.
Run publish only:
```bash
narratio publish 2026-04-04
```
Equivalent command:
Equivalent:
```bash
narratio run-stage publish 2026-04-04 --force
```
Publish uploads:
Publish commit model:
- run history files under `{session_prefix}/runs/{run_id}/` (excluding `audio/`)
- configured published outputs from `pipeline.publish.outputs`
- `previous/**` cache files when present
- `current/manifest.json`
- `current/run_id.txt` last
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
audio and the run-local Notarius staging bundle;
- uploads configured published outputs and `previous/**` cache files into the
same immutable run scope, including only explicitly configured extraction
lanes;
- writes `{session_prefix}/runs/{run_id}/commit.json` after all declared
immutable objects are uploaded and verified; and
- writes `{session_prefix}/current/commit-pointer.json` once, last.
`current/run_id.txt` is the remote commit marker.
`current/commit-pointer.json` is the remote current-state commit marker. It
selects exactly one immutable commit, which declares the complete object set.
## Published Outputs and Locks
## Remote Commit Migration
Published output behavior:
The immutable remote commit contract uses
`runs/{run_id}/commit.json` to declare a run's complete object set and a small
`current/commit-pointer.json` to select it. The pointer binds the selected
commit by version, checksum, size, and storage generation; committed artifacts
are also checksum- and generation-bound. Readers accept this contract now and
strictly reject mismatched or unknown data.
- outputs are source-based rules in `pipeline.publish.outputs`.
- required missing unlocked sources fail publish.
- optional missing unlocked sources are skipped.
- selected artifacts (`--artifacts`) only filter configured `narratio.artifact.<key>` output rules.
- built-in transcript and bounds output rules are not filtered by `--artifacts`.
Legacy reads are limited to a coherent `current/manifest.json` and
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
write that pair and remote commit state does not carry local
`current_pointer_written` metadata.
Lock behavior:
## Publish Locks
- static locks: `pipeline.publish.locks`.
- mutable locks: `{session_prefix}/locks.yml`.
- effective lock set is static + mutable; static wins on duplicate sources.
- locked outputs are intentional skips and do not fail publish.
- lock commands mutate only remote mutable locks.
Lock sources:
- static locks in `pipeline.publish.locks`
- mutable remote locks in `{session_prefix}/locks.yml`
Effective lock rules:
- static and remote locks are merged;
- static locks win on source collisions;
- locked outputs are intentional skips;
- lock add/remove commands mutate only remote lock state through generation-bound
conditional writes. A command retries a bounded number of concurrent
conflicts while its invocation context remains active, so it never replaces a
different lock-document generation; and
- a publish re-reads remote locks immediately before it writes the current
commit pointer. A lock committed before that recheck prevents selecting the
new snapshot, even though its already-uploaded immutable objects may remain
available for a later retry.
Examples:
```bash
narratio session locks 2026-04-04
narratio session locks add 2026-04-04 narratio.artifact.session_recap --reason "manual edits" --force
narratio session locks remove 2026-04-04 narratio.artifact.session_recap
```
## Restore Workflow
Use restore when local durable session state is missing/stale and committed remote current state is authoritative.
Use restore when local durable session state is missing or stale and remote committed current state is authoritative.
Preview:
Dry run:
```bash
narratio session restore 2026-04-04 --dry-run
@@ -78,26 +277,37 @@ Apply:
narratio session restore 2026-04-04
```
`--dry-run` does not write durable session files. It still reads the selected
remote current state and may read object identity/content needed to classify the
plan, so it is not a network-free operation.
Default restore scope:
- `manifest.json`
- `transcripts/**`
- `artifacts/**`
- `previous/**` when required by configured previous-session artifact inputs
- the committed session manifest and the committed transcript/artifact objects
declared by the selected remote commit
- `previous/**` when needed by configured previous-session artifact inputs
Optional:
- add `--include-audio` to restore `audio/**`.
- `--include-audio` to include `audio/**`
- `--force` to overwrite eligible conflicting regular files; it never replaces
directories or other non-regular local targets
Restore reads committed current state only (`current/run_id.txt`, `current/manifest.json`).
Restore writes an execution report at `reports/restore-latest.json`.
## Workspace and State Layout
If restore fails after beginning installation, it leaves a durable
`.restore-incomplete.json` marker in the session root. Pipeline runs will stop
until you rerun the same restore command and it completes. Restore intentionally
does not try to roll back files already installed; retrying the selected remote
snapshot is the recovery procedure.
## Local State Layout
Session root:
- `{workspace.root}/work/{campaign}/{session_id}/`
- `{workspace.root}/work/{campaign}/{session_id}`
Durable session state:
Durable session paths:
- `manifest.json`
- `inputs/**`
@@ -110,61 +320,97 @@ Durable session state:
- `config/**`
- `runs/**`
Run-local stage layout:
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
stderr, and pre-promotion output remain in the producing run's `extract`
directory as described in [Extraction Workflow](#extraction-workflow).
- `runs/{run_id}/{stage}/outputs|logs|reports|config|scratch`
Run-local layout:
Stages typically write run-local outputs first, then materialize canonical session outputs on success.
- `runs/{run_id}/{stage}/outputs`
- `runs/{run_id}/{stage}/logs`
- `runs/{run_id}/{stage}/reports`
- `runs/{run_id}/{stage}/config`
- `runs/{run_id}/{stage}/scratch`
## Resume and Force Rules
Spool layout (runtime/transient):
- `run` and `run-stage` skip succeeded stages unless `--force` is set.
- `resume` starts at the first non-succeeded stage.
- force-rerunning an upstream succeeded stage marks downstream succeeded stages as `stale`.
- `--force` does not bypass publish locks.
- `{spool.root}/{campaign}/{session_id}/{run_id}/...`
- restore audio spool under `{spool.root}/{campaign}/{session_id}/restore/audio`
Cache layout (durable S3 audio cache):
- `{cache.root}/s3/{bucket}/...`
Each cached audio file has an adjacent managed identity record. It binds the
file to its remote object version and verified digest; deleting or altering the
record simply causes Narratio to download and verify the object again.
### Workspace Permissions
Ordinary Narratio workspace content is intentionally shareable with the
workspace group. On POSIX systems, Narratio-created workspace, spool, and cache
directories converge on setgid `02775`; ordinary files, including manifests,
transcripts, generated configuration, logs, reports, and Notarius artifacts,
converge on `0664`. Narratio explicitly applies these modes so a restrictive
caller umask does not remove group write or setgid. It does not change file or
directory ownership: the configured workspace's existing group is inherited.
Windows does not implement POSIX mode bits or setgid semantics. Configure the
workspace, spool, and cache locations with an ACL that grants the collaborating
group read/write access, and configure credential locations with an ACL limited
to the intended credential owner. Do not use POSIX mode displays as evidence of
Windows access control.
API keys are credentials, not ordinary workspace data. Store them outside the
shared workspace or in a separately restricted credential location; ordinary
workspace group access must never be treated as authorization to read keys.
On POSIX, provision a credential directory as `0700` and credential files as
`0600`; Narratio rejects group- or other-readable configured credential paths.
On Windows, restrict the directory and files with ACLs to the credential owner.
External adapter results are individually bounded before Narratio validates or
materializes them. These per-file limits do not reserve disk space: prevent hard
disk exhaustion with filesystem, service, container, or volume quotas sized for
the session workload.
## Cleanup
Automatic post-publish cleanup is considered only when publish executes successfully and commits current state.
Config toggles:
- `pipeline.spool.delete_audio_after_publish=true`
- `pipeline.workspace.cleanup_after_publish=true`
Manual cleanup:
Session-scoped cleanup:
```bash
narratio clean 2026-04-04
```
Global cleanup:
```bash
narratio clean --all
```
Cache is preserved by default. Use `--clear-cache` to remove matching S3 audio cache entries.
## Failure and Recovery
After stage failure, Narratio keeps manifests and run-local files for inspection.
Standard recovery flow:
1. inspect status:
Dry-run and cache variants:
```bash
narratio session status 2026-04-04
narratio clean 2026-04-04 --dry-run --clear-cache
narratio clean --all --dry-run --clear-cache
```
2. if needed, inspect restore plan:
Rules:
```bash
narratio session restore 2026-04-04 --dry-run
```
3. fix root cause.
4. continue with `resume`, or rerun a stage with `--force` then `resume`.
- `clean` deletes work/spool session state;
- cache is preserved unless `--clear-cache` is set;
- each deletion is confined beneath its configured workspace, spool, or cache
root and refuses symlinked paths;
- automatic post-publish cleanup is gated by successful publish commit plus:
- `pipeline.spool.delete_audio_after_publish=true`
- `pipeline.workspace.cleanup_after_publish=true`
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
reports incomplete, the remote committed snapshot remains current; rerun
Narratio to retry only the outstanding confined local cleanup.
## Operational Caveats
- local and S3 audio modes are mutually exclusive.
- publish requires prerequisite stages through `analyze` to be `succeeded`.
- restore requires configured object storage and committed current state.
- `session status` and `session artifacts --remote` both report remote published-output availability when storage is configured.
- Local and S3 audio modes are mutually exclusive.
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
- Restore requires configured object storage and committed remote current state.
- Storage-backed commands load filesystem secrets before object-store initialization.

272
docs/policy/architecture.md Normal file
View File

@@ -0,0 +1,272 @@
# Architecture
This document defines Narratio's intended high-level architecture and the
invariants that changes must preserve. Implemented component details belong in
the [Internal Overview](../internal/overview.md) and its linked documents.
Significant architectural decision history belongs under `docs/adr/` when such
records exist.
## System Shape
Narratio is a small Go application that turns D&D session audio into polished
transcripts and generated session artifacts. It is an explicit, stage-driven
orchestrator, not a general workflow engine.
Narratio coordinates specialized external systems rather than reimplementing
their domains:
- WhisperX performs transcription;
- Seriatim performs deterministic transcript processing and rendering;
- Audita performs transcript correction and polishing;
- Notarius extracts validated structured artifact bundles; and
- Scriptorium executes prompts and produces configured artifacts.
Narratio owns orchestration, configuration resolution, session and run state,
artifact and path modeling, manifest persistence, stage sequencing, resume,
restore, cleanup gates, and publish semantics. External contracts are defined
in the [integration documentation](../integrations/).
The pipeline has one canonical ordered stage set. Configuration may enable,
disable, or parameterize supported behavior, but it must not turn that sequence
into an arbitrary DAG or hide orchestration in generic workflow abstractions.
The implemented stage inventory belongs in the
[Internal Overview](../internal/overview.md).
Narratio is contract-first without being abstraction-heavy. Interfaces and
extension points should protect demonstrated boundaries. New abstraction is not
itself an architectural goal.
## Ownership And Dependency Direction
The application boundary owns command dispatch, configuration selection,
production composition, session locking, and top-level lifecycle. It may depend
on concrete implementations to assemble a run.
Stage orchestration expresses intent in Narratio-level data and interfaces.
Stages may depend on configuration, manifest, artifact, path, and adapter
contracts, but they must not depend on transport-specific request types,
subprocess argument construction, cloud SDK types, or downstream tool internals.
Adapters translate between Narratio contracts and external systems. They own
HTTP, subprocess, notification, and object-storage mechanics, including command
construction, transport behavior, provider response handling, and external
error adaptation. External dependency types must remain inside the adapter that
owns them unless that dependency is the adapter's explicit public contract.
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
construction, notification transport, and object-storage SDK details remain
behind these boundaries.
State and path services must not infer stage policy. Storage implementations
receive explicit bucket-relative keys and do not infer campaign, session, run,
or root-prefix semantics. Manifest persistence records transitions but does not
choose orchestration policy. Artifact resolution identifies and validates
artifacts but does not execute producers.
Dependencies should remain narrow and point toward Narratio-owned contracts.
Prefer the Go standard library. Add an external dependency only when it provides
a clear correctness, security, interoperability, or complexity benefit, and
confine it to the boundary that needs it.
## Stage Boundaries
Each stage has one explicit responsibility and declares:
- required input state;
- produced output state;
- configuration it consumes;
- external adapters it uses;
- manifest references and metadata it reads or writes;
- skip, force, invalidation, and resume behavior; and
- failure behavior.
Stages write and validate run-local results before materializing canonical
outputs where that distinction applies. A stage is complete only after its
required outputs have been written, validated, and recorded in durable manifest
state. Later stages depend on recorded success and artifact resolution, not
merely on incidental files existing on disk.
A failed or interrupted stage must not be presented as successful. Failure
should preserve enough local state and diagnostics for inspection, recovery,
and resume. Forcing an upstream stage invalidates succeeded downstream work
according to the canonical stage order.
A stage may explicitly self-skip with a stable reason and no outputs. That
outcome is persisted, clears older outputs owned by the stage, and is
reconsidered on a later invocation. A stage may also validate whether an
otherwise successful recorded result is still resumable; an obsolete result
is staled and rerun, while an unsafe condition that prevents a sound decision
stops execution.
Shared behavior should live behind a narrow service or helper with one clear
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
artifact, or path policy ad hoc.
## Manifest, Resume, And Restore
The session manifest is the durable ledger for progress across invocations. It
records session and run identity, stage state, input and output references,
diagnostic references, checksums or provenance where useful, and non-secret
adapter and publish metadata.
Resume and skip decisions are manifest-driven. Filesystem state may be
inspected and validated, but file presence alone does not replace recorded
stage state. Invocation-scoped run records provide an audit of one execution;
they do not replace the session manifest as progress authority.
Restore treats committed remote current state as its authority. It must plan
deterministically, confine remote-to-local paths, protect local conflicts, and
install the validated session manifest after other restored durable files. The
physical workflow and recovery procedures belong in
[Operations](../operations.md).
Restore and runner transitions for one session use the same local lock. A
durable incomplete-restore marker blocks runner reuse after a partial restore;
safe retry, rather than rollback of arbitrary local effects, is the recovery
mechanism. Restored manifest-local references must be confined to the selected
local session root, never trusted as producer-machine absolute paths.
For the immutable remote-commit protocol, a restore or status operation binds
to one pointer-selected commit and only its declared object identities. A force
flag may replace an eligible regular managed file, but never turns a directory
or other non-regular conflict into a successful restore.
## Configuration
Configuration is strict, explicit, centralized, and operator-oriented.
- YAML decoding rejects unknown fields.
- Defaults are centralized and testable.
- Empty configured values do not silently replace meaningful defaults.
- Validation rejects invalid composition before stage execution where
practical.
- Session templating remains narrow and deterministic rather than becoming a
general configuration language.
- Secret values are supplied indirectly and are not persisted in ordinary
configuration.
Narratio must not become a second configuration system for downstream tools.
External systems own their runtime defaults wherever practical; Narratio passes
the paths required by its stage contracts and explicit operator overrides. The
field-level contract and credential-supply mechanisms belong in
[Configuration](../config.md).
## Artifacts, Paths, And Storage
Artifact identities and local and remote paths are application contracts.
Canonical helpers own workspace, spool, cache, session, run, input, transcript,
artifact, log, report, configuration, and publish-current paths. Callers must
not reconstruct canonical paths through scattered string concatenation.
Reusable audio cache entries require a typed record that binds a confined,
no-follow regular file and its digest to the selected remote object identity.
Size alone and unqualified multipart ETags are not content-integrity evidence.
Artifact resolution is deterministic and manifest-aware. Producers materialize
canonical outputs before reporting success, and consumers resolve declared
artifact identities rather than infer files from unrelated directory contents.
External artifact bundles become current only through validated immutable
promotion and manifest records; directory presence alone never establishes
availability.
Writes, moves, replacements, and deletions must use narrow, explicit,
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
relative destinations must not expand the scope of an operation. Cleanup is
permitted only through explicit operator action or configured post-publish
gates, and it must preserve durable cache unless cache removal is explicitly
requested.
Physical layout, retention, and operational lifecycle belong in
[Operations](../operations.md). Logical external formats and durable integration
contracts belong under [Integrations](../integrations/).
## Publish Commit Boundary
Publish has one explicit remote commit boundary. A remote run becomes current
only after Narratio has successfully uploaded its immutable run-scoped objects,
the immutable commit manifest, and finally the current commit pointer.
`current/commit-pointer.json` is the sole mutable selector and must be written
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
must not be presented as current remote state. Publish locks remain authoritative
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
generation preconditions and are revalidated immediately before pointer
selection; loss of that check leaves the prior committed snapshot current.
Automatic local cleanup is permitted only after a successful publish commit,
only when explicitly configured, and only through the path-safety guardrails.
It is a durable local obligation bound to that committed run and its exact
targets, not an inferred side effect of the current stage list. A cleanup
failure makes the invocation incomplete while leaving the committed remote
snapshot authoritative; later invocations resume the recorded obligation.
## Security, Privacy, And Diagnostics
Narratio distinguishes ordinary workspace data from credentials. Campaign and
session material—including manifests, transcripts, prompts, generated
configuration, logs, reports, diagnostics, and Notarius artifacts—is
intentionally shareable with the configured workspace group. API-key material
is sensitive and is not covered by the ordinary workspace-sharing policy.
On POSIX systems, Narratio-created ordinary workspace directories converge on
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
umask is restrictive. This preserves the existing workspace group for nested
creation and atomic replacements without changing ownership. API-key storage
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
are not authoritative; operators must provide the equivalent shared-group and
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
to manifests, logs, generated configuration, reports, publish metadata,
documentation, or examples. Secrets enter through configured environment
variable names or secret-file references. Diagnostics should avoid transcript
and prompt content unless a deliberate, bounded inspection mechanism requires
it.
Logs, reports, generated invocation files, generated configuration, and render
debug files are diagnostics, not canonical pipeline products. They should be
durable and discoverable where configured, and manifest references must preserve
the distinction between diagnostics and artifacts.
Documentation security rules belong in the
[Documentation Policy](documentation.md). Credential supply belongs in
[Configuration](../config.md), while permissions, sensitive runtime-artifact
handling, and recovery belong in [Operations](../operations.md).
## Determinism And Testability
Narratio prefers deterministic behavior where practical, including stable local
and remote layouts, sorted operation order, predictable generated
configuration, repeatable command construction, deterministic artifact
resolution, and reproducible planning.
Run IDs and timestamps may be intentionally variable, but surrounding behavior
must remain controllable in tests. Core behavior should be testable without live
external services; expensive, nondeterministic, destructive, or external
boundaries should be replaceable with focused test doubles. General testing
philosophy and sufficiency rules belong in the [Testing Policy](testing.md).
## Documentation And Decision Records
Documentation follows the [Documentation Policy](documentation.md). Current
behavior belongs in its canonical user, operator, integration, architecture, or
internal owner. Proposed behavior and implementation status belong under
`docs/roadmap/`.
Significant architectural decisions may be recorded under `docs/adr/` using the
format and lifecycle defined by the documentation policy. ADR acceptance does
not establish that a decision has been implemented.
## Architectural Non-Goals
Narratio does not aim to provide:
- a generic DAG or workflow engine;
- a replacement configuration layer for WhisperX, Seriatim, Audita,
Scriptorium, or other downstream tools;
- a storage abstraction broader than the needs of this pipeline;
- stage logic coupled directly to cloud SDKs, transports, subprocess details,
or downstream implementation internals;
- raw-secret persistence;
- implicit cross-stage behavior that bypasses manifest and artifact contracts;
or
- a prompt-authoring system.

View File

@@ -0,0 +1,148 @@
# Documentation Policy
## Purpose
This policy assigns each documentation topic to one canonical owner. Its goal is
to keep Narratio documentation accurate, concise, discoverable, and resistant
to drift for users, operators, developers, integrators, and LLM coding agents.
## Core Rules
### One Canonical Owner
Each authoritative fact belongs in one document. A non-owning document may give
a short, stable summary for orientation, but it must link to the canonical owner
instead of repeating volatile details.
Volatile details include commands, flags, configuration fields and defaults,
stage or integration keys, schemas, file names, paths, status codes, retry
behavior, and runtime guarantees. If readers could reasonably treat a statement
as a contract, maintain it only in the owning document.
### Current And Future Behavior
Outside `docs/roadmap/`, documentation describes implemented behavior only.
Partial features may be described only to their implemented boundary.
ADRs are the narrow exception: an ADR may record an accepted architectural
decision before implementation, but acceptance must not be presented as proof
that the behavior exists. The roadmap owns implementation status and sequencing
until the decision is implemented. Current architecture, user, operator,
integration, and internal documentation are updated when the behavior lands.
### Audience And Detail
Write for the document's stated audience and include only the detail needed for
its owned topic. User and operator docs should not expose implementation detail.
Developer docs should link to user-facing and external contracts rather than
restate them.
### Examples
Complete copyable files belong in `examples/`. Documentation may use the
smallest illustrative snippet needed to explain its owned topic, but should link
to maintained examples instead of embedding a second complete copy.
Examples must be valid, secret-free, and tested where practical. Commands and
configuration used in documentation should match the application.
### Security And Privacy
Documentation and examples must not contain real credentials, private keys,
private environment dumps, sensitive source material, or private infrastructure
details unless intentionally public. Document secret-handling mechanisms, not
secret values.
## Canonical Ownership
| Topic | Canonical owner | Owned content | Content owned elsewhere |
| --- | --- | --- | --- |
| Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, output conventions, and exit behavior. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, stage implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schemas, fields, defaults, environment overrides, validation rules, and user-selectable stage or integration settings. | Complete example files, CLI syntax, runtime state lifecycle, implementation details. |
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and remote-state layout, output and diagnostic handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical artifact schemas, implementation mechanics. |
| Troubleshooting | `docs/troubleshooting.md` | Symptom-driven diagnosis, likely causes, safe inspection steps and remedies, and links to relevant contracts. | CLI syntax, configuration definitions, operational procedures, integration contracts, implementation mechanics. |
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical artifact paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
Documents that do not exist are required only when the corresponding interface
or responsibility exists. Do not create placeholder API, consumer, integration,
or operations documents for behavior the application does not have.
## Boundary Rules
### Orientation
The README owns product orientation. The developer guide routes contributors.
Architecture owns normative structure. Internal overview owns the current
concrete component map. These documents may link to one another but should not
maintain parallel package or behavior descriptions.
### Commands, Configuration, Operations, And Troubleshooting
CLI documentation answers how to invoke the application. Configuration
documentation answers what settings mean. Operations answers what happens to
runtime state and how to operate or recover the application. Troubleshooting
starts from observable symptoms and links readers to the owning command,
configuration, operational, or integration contract. When a workflow crosses
these topics, choose the document that owns the task and link to the other
contracts.
### Contracts And Implementation
Integration and API documents define externally observable shapes and
semantics. Internal documents explain how Narratio implements or consumes those
contracts. Internal docs may name a field, file, or protocol to identify a
dependency, but must link to its canonical contract for the definition.
### Security Topics
This policy owns what documentation and examples may contain. Architecture owns
application security invariants. Configuration owns credential-supply
mechanisms. Operations owns permissions and handling of sensitive runtime
artifacts. Troubleshooting owns safe diagnostic and remediation guidance.
Internal docs own implementation mechanisms only.
## Architecture Decision Records
Use sequentially numbered ADR filenames such as
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
1. title;
2. status;
3. date;
4. context;
5. decision;
6. alternatives considered;
7. consequences.
Treat the decision content of an accepted ADR as immutable. When a decision
changes, create a new ADR and update the earlier ADR's status to superseded.
Rejected architectural alternatives belong in the ADR; rejected product ideas
belong in the roadmap.
## Maintenance
When behavior changes, update its canonical owner in the same change. If
ownership moves, remove the old definition and replace it with a link where
navigation remains useful.
Before completing documentation work:
- verify affected behavior and examples;
- check commands, flags, fields, defaults, schemas, and paths against their
implementation;
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
- remove stale references and validate links;
- confirm that non-owning documents summarize and link rather than redefine;
- confirm that no secrets or sensitive private data were added.

296
docs/policy/testing.md Normal file
View File

@@ -0,0 +1,296 @@
# Testing Policy
## Purpose
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
## Every test has a cost
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
A test must be:
- written and reviewed;
- understood by future maintainers and coding agents;
- executed in local and CI workflows;
- diagnosed when it fails;
- updated when legitimate behavior changes;
- maintained as fixtures, APIs, and dependencies evolve; and
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
A test is warranted only when the confidence it provides justifies these costs.
Apply this cost-benefit analysis at two levels:
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
## Default testing style
Use a **classical/Detroit-style** approach:
- Test observable behavior, resulting state, contracts, and invariants.
- Use real internal collaborators when they are fast and deterministic.
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
## Test execution requirements
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
## What deserves tests
Prioritize tests for:
1. Public and package-level contracts.
2. Domain rules and important invariants.
3. Boundary conditions and malformed input.
4. Failure handling, cancellation, retries, recovery, and partial success.
5. Serialization, schemas, compatibility, and round trips.
6. Previously observed or plausible regressions.
7. Representative integration and end-to-end workflows.
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
## Choose the right test boundary
Test through the narrowest stable boundary that expresses the behavior clearly.
This is often the package API, but it may instead be:
- a smaller pure function when dense domain logic is most clearly isolated there;
- a package-level operation when several internal collaborators jointly produce the behavior; or
- a larger integration boundary when correctness emerges from interaction with a real dependency.
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
## Test behavior, not implementation
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
Before adding or retaining a test, ask:
> What realistic defect would this test catch?
A test is suspect when its main purpose is to detect that someone:
- changed an internal constant;
- renamed or split a private helper;
- reordered equivalent internal operations;
- changed incidental formatting;
- replaced one correct algorithm with another; or
- refactored internal object structure without changing behavior.
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
## Expected effects of different changes
Use the following expectations when evaluating test failures and test maintenance:
| Change | Expected effect on tests |
|---|---|
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
## Separate mechanism from policy
Configurable thresholds and defaults must not be duplicated throughout the test suite.
For example, do not encode an internal concurrency limit indirectly:
```go
// Production policy:
const maxConcurrency = 4
// Brittle test:
err := startProcesses(5)
require.Error(t, err)
```
Instead, test the mechanism relationally:
```go
const limit = 2
runner := NewRunner(limit)
require.NoError(t, runner.Start(limit))
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
```
The test should prove:
- the configured limit is accepted; and
- one beyond the configured limit is rejected.
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
For concurrency limits, test both kinds of behavior when relevant:
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
## Avoid semantic duplication across layers
Each behavior should have a clear test owner.
- Parser tests own parsing cases.
- Validator tests own validation rules.
- Domain tests own transformations and invariants.
- Adapter tests own external integration behavior.
- Orchestrator tests own coordination and failure propagation.
- CLI tests own argument and configuration mapping.
- End-to-end tests prove that representative assembled workflows work.
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
## Use test doubles deliberately
Choose the least elaborate test double that provides the required control or observation.
As a default:
1. Prefer real collaborators when they are fast and deterministic.
2. Use small in-memory fakes when realistic stateful behavior is helpful.
3. Use stubs when a dependency only needs to provide controlled responses.
4. Use mocks when the interaction itself is contractual.
Mocks are appropriate when the contract includes facts such as:
- a notification is sent exactly once;
- a transaction is committed only after successful writes;
- cancellation reaches a subprocess;
- an expensive API is called no more than once; or
- a security audit event is emitted.
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
## Go-specific guidance
Use:
- table-driven tests for meaningful behavioral categories and boundaries;
- `t.TempDir()` for real filesystem behavior;
- `httptest.Server` for realistic HTTP interactions;
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
- golden files only when the complete output is intentionally stable;
- integration tests where correctness depends on component interaction; and
- a small number of representative end-to-end tests.
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
## Coverage
Coverage is a diagnostic, not a target.
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
## Regression tests
A bug fix should normally include a regression test that fails before the fix and passes afterward.
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
## Deleting or rewriting tests
Tests are maintained code, not permanent historical artifacts.
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
Strong candidates include tests that:
- require updates after harmless internal changes;
- directly assert private constants without protecting a real contract;
- duplicate the same policy across several layers;
- verify mock choreography rather than outcomes;
- snapshot large amounts of incidental output;
- test trivial private helpers already exercised through stable package behavior;
- protect risks already covered more effectively elsewhere;
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
- no longer correspond to a plausible failure mode.
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
## Reviewing a proposed test
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
1. What realistic defect would it catch?
2. How likely is that defect?
3. How consequential would it be?
4. Is the behavior already protected elsewhere?
5. At which layer should this behavior be owned?
6. Does the test assert a durable contract or an incidental implementation detail?
7. Could the implementation be refactored without changing the behavior and without editing this test?
8. What should cause this test to fail?
9. What legitimate changes should not cause this test to fail?
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
11. Is there a smaller or more direct test that protects the same risk?
Do not add the test when its expected lifetime cost exceeds its expected protective value.
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
## Definition of sufficient
A test suite is sufficient when:
- important contracts and invariants are protected;
- meaningful boundaries and failure modes are exercised;
- realistic and consequential regressions are credibly protected against silent recurrence;
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
- important external boundaries have realistic integration coverage;
- representative complete workflows are tested;
- failures provide useful signal rather than redundant noise;
- legitimate internal changes usually do not require test edits; and
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
The governing rule is:
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.

View File

@@ -1,231 +0,0 @@
# Roadmap: Campaign Registry
Status: Implemented
## Problem
Narratio currently treats campaign configuration as one selected
`campaign.yml` file:
- command flags use `--campaign <path>`;
- default discovery searches fixed system file locations;
- `campaign.yml` uses `campaign:` as the identity field.
That model works for a single campaign, but it is awkward for installations
that manage multiple campaigns. Operators need to pass file paths or maintain a
single global campaign config, while the newer session-oriented CLI already
uses concise positional session IDs and remote session lookup.
The campaign selection model should become ID-based and pipeline-owned.
Pipeline config should describe where campaigns live, commands should select a
campaign by ID, and each campaign directory should contain its stable campaign
materials.
## Target Model
`pipeline.yml` owns the campaign registry:
campaigns:
root: /usr/local/share/narratio/campaigns
default_campaign_id: dilfs
Campaign files live at the conventional path:
{campaigns.root}/{campaign_id}/campaign.yml
The first implementation should use only the conventional path. Recursive
discovery of every `campaign.yml` under `campaigns.root` is deferred to a
future stage.
Each campaign file uses `campaign_id` as the canonical identity field:
campaign_id: dilfs
session_template_file: ./session.template.yml
inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
Campaign-relative files continue to resolve relative to the selected
`campaign.yml`, including stable input files and `session_template_file`.
The public CLI changes from path-based campaign selection to ID-based campaign
selection:
- `--campaign <id>` selects a campaign ID.
- `--campaign-file <path>` explicitly loads one campaign file for
development, tests, and unusual local workflows.
- `--campaign` and `--campaign-file` are mutually exclusive.
If neither `--campaign` nor `--campaign-file` is passed, Narratio uses
`pipeline.campaigns.default_campaign_id`. If no campaign can be selected,
commands fail clearly before session loading or stage execution.
Resolved campaign ID remains the campaign segment used for:
- workspace paths;
- spool paths;
- S3 session prefixes;
- remote `session.yml` lookup;
- archive locks and promoted output keys;
- session/campaign mismatch validation;
- status, plan, restore, and helper output.
## Compatibility Policy
This is a breaking public/config contract change.
After the cutover:
- `--campaign` no longer accepts a filesystem path;
- default fixed campaign file discovery is removed;
- `campaign:` is no longer accepted in `campaign.yml`;
- `campaign_id:` is required.
Keep `--campaign-file` as the only explicit file override. Do not retain hidden
aliases for the old `--campaign <path>` behavior.
## Implementation Stages
### Stage 1: Add Campaign Registry Selection
Status: Implemented
Add the registry model and switch command loading to resolve campaigns through
pipeline config.
Implementation requirements:
- Add `pipeline.campaigns.root`.
- Add `pipeline.campaigns.default_campaign_id`.
- Add `campaign_id` to campaign config and make it the canonical identity.
- Resolve pipeline config first, then campaign selection.
- Use this selection order:
1. explicit `--campaign-file <path>`;
2. explicit `--campaign <id>`;
3. `pipeline.campaigns.default_campaign_id`;
4. fail clearly.
- For ID selection, load `{campaigns.root}/{campaign_id}/campaign.yml`.
- Validate that the loaded `campaign_id` matches the selected ID.
- Reject `--campaign` with `--campaign-file`.
- Preserve strict YAML decoding.
- Preserve campaign-relative stable input and session template resolution.
- Keep storage details behind the existing storage adapter and object-store
helper.
- Keep remote session lookup and archive key construction based on the
resolved campaign ID.
Acceptance criteria:
- Commands can run with only a pipeline config and the pipeline default
campaign ID.
- Commands can select another campaign with `--campaign <id>`.
- Commands can load a specific file with `--campaign-file <path>`.
- Existing session loading, remote session fallback, prepare materialization,
restore, archive, locks, clean, analyze, and publish behavior continue to use
the same resolved campaign identity.
- No generic config registry framework is introduced.
### Stage 2: Remove Old Single-File Campaign Behavior
Status: Implemented
Remove the old public campaign file model after registry selection is in
place.
Implementation requirements:
- Remove fixed default campaign config discovery from command loading.
- Remove `DefaultCampaignConfigSearchPaths` and related path-only resolution if
no current tests or helpers still need them.
- Remove support for `campaign:` from `campaign.yml`.
- Update validation errors to refer to `campaign_id`.
- Update examples to use campaign directories and `campaign_id`.
- Update current-behavior docs to document:
- `pipeline.campaigns.root`;
- `pipeline.campaigns.default_campaign_id`;
- `campaign_id`;
- `--campaign <id>`;
- `--campaign-file <path>`.
- Update troubleshooting examples that currently pass `--campaign <path>`.
Acceptance criteria:
- `campaign.yml` files with `campaign:` fail strict decoding.
- `--campaign /path/to/campaign.yml` is treated as a campaign ID and fails
unless that ID exists under `campaigns.root`.
- `--campaign-file /path/to/campaign.yml` is the supported file override.
- User-facing docs no longer describe fixed campaign config discovery.
## Test Guidance
Focused tests:
- `go test ./internal/config -v`
- `go test ./internal/app -v`
- `go test ./internal/stage -run Prepare -v`
Full validation:
- `go test ./...`
Config tests to add or update:
- strict decode accepts `pipeline.campaigns.root`;
- strict decode accepts `pipeline.campaigns.default_campaign_id`;
- strict decode accepts `campaign_id`;
- selected campaign ID mismatch fails;
- missing campaign root fails when ID selection is needed;
- missing default campaign ID fails when no explicit campaign selector is
passed;
- old `campaign:` fails after Stage 2.
App tests to add or update:
- `--campaign <id>` resolves `{campaigns.root}/{id}/campaign.yml`;
- omitted `--campaign` uses `pipeline.campaigns.default_campaign_id`;
- `--campaign-file` loads an explicit campaign file;
- `--campaign` plus `--campaign-file` fails;
- remote session fallback uses the resolved campaign ID;
- `session init`, `run`, `run-stage`, `resume`, `analyze`, `publish`, `clean`,
and `session` subcommands all use the same campaign selection path;
- path-based `--campaign` examples and tests are removed after Stage 2.
## Documentation Guidance
Update current-behavior docs only after implementation lands:
- `docs/config.md`
- `docs/cli.md`
- `docs/operations.md`
- `docs/troubleshooting.md`
- relevant files under `docs/internal/`
- `examples/`
Planned campaign registry behavior belongs only in this roadmap until the code,
tests, examples, and current-behavior docs are updated.
## Architecture Guardrails
- Keep Narratio explicit and stage-driven.
- Do not introduce a generic configuration registry or workflow framework.
- Keep YAML decoding strict.
- Keep defaults centralized and testable.
- Keep campaign-relative path resolution centralized.
- Use centralized S3 and workspace path helpers.
- Keep storage details behind `storage.ObjectStore`.
- Keep secret-backed object-store construction in `internal/app`.
- Preserve manifest-driven resume and restore behavior.
- Do not store raw secrets in campaign configs, manifests, logs, generated
configs, or archive metadata.
## Assumptions
- The canonical pipeline schema is grouped under `campaigns`.
- The canonical campaign identity field is `campaign_id`.
- `--campaign` means campaign ID.
- `--campaign-file` is retained as an explicit override.
- Recursive discovery is planned but not part of the first implementation.
- Existing production configs can be migrated from `campaign:` to
`campaign_id:` and from `--campaign <path>` to `--campaign <id>` or
`--campaign-file <path>`.

View File

@@ -1,159 +0,0 @@
# Roadmap: Legacy Config Cleanup
Status: Implemented
## Problem
Narratio's current pipeline config schema still accepts fields that predate the current storage, artifact, and previous-session models:
- `pipeline.storage.bucket`
- `pipeline.storage.prefix`
- `pipeline.analyzer.*`
- `previous_session_artifact`
These names make the config reference harder to trust because they suggest supported behavior that operators should no longer use. The modern interface is:
- `pipeline.storage.s3.*` for remote storage.
- Scriptorium configured artifacts under `pipeline.scriptorium.artifacts`.
- Canonical artifact source IDs such as `narratio.artifact.<configured_artifact_key>`.
- Canonical previous-session artifact sources such as `narratio.previous_session.artifact.<configured_artifact_key>`.
Strict YAML decoding should reject removed legacy fields once this cleanup lands.
## Current State
`pipeline.storage.bucket` and `pipeline.storage.prefix` were inert compatibility fields and have been removed:
- They are no longer present on `config.StorageConfig`.
- Strict decoding rejects them.
- Runtime S3 behavior uses `pipeline.storage.s3.bucket` and `pipeline.storage.s3.root_prefix`.
- No current code reads the top-level storage bucket or prefix fields.
`pipeline.analyzer.*` was legacy code surface and has been removed:
- `config.PipelineConfig` no longer includes analyzer config.
- Strict decoding rejects `pipeline.analyzer`.
- `stage.Env` no longer exposes an analyzer runner, and `internal/adapters/analyzer` has been deleted.
- Modern analyze execution is Scriptorium-backed; the analyzer adapter is not used by current stage execution.
`previous_session_artifact` was a live legacy behavior and has been removed:
- Config validation rejects it as an unsupported Scriptorium input source.
- The analyze stage no longer has path-based previous-artifact resolution through `inputs.<name>.path`.
- Tests cover canonical previous-session sources and the rejection of the legacy source.
- The canonical replacement is `narratio.previous_session.artifact.<configured_artifact_key>`, resolved through the previous-session cache/catalog model.
## Target Model
The pipeline config schema should expose only current behavior:
- Remote storage is configured only through `pipeline.storage.s3.*`.
- Generated artifacts are configured only through `pipeline.scriptorium.artifacts`.
- Scriptorium artifact inputs use canonical source IDs.
- Previous-session artifact inputs use `narratio.previous_session.artifact.<configured_artifact_key>`.
- Unknown legacy fields fail strict YAML decoding.
No compatibility aliases should remain unless a future migration requirement explicitly reintroduces them.
## Cleanup Order
### Stage 1: Remove Inert Storage Compatibility Fields
Status: Implemented
Remove `pipeline.storage.bucket` and `pipeline.storage.prefix`.
Implementation requirements:
- Delete `StorageConfig.Bucket` and `StorageConfig.Prefix`.
- Keep `StorageConfig.Backend` and `StorageConfig.S3`.
- Confirm all runtime storage paths continue to use `storage.s3.bucket` and `storage.s3.root_prefix`.
- Update examples and docs to remove top-level storage `bucket` and `prefix`.
- Add or update strict-decode tests proving `pipeline.storage.bucket` and `pipeline.storage.prefix` are rejected.
Acceptance criteria:
- Existing S3 workflows still pass with `pipeline.storage.s3.bucket`.
- Pipeline configs containing top-level `storage.bucket` or `storage.prefix` fail to load.
- No docs or examples present those fields as available.
### Stage 2: Remove Legacy Analyzer Schema and Adapter Surface
Status: Implemented
Remove the unused analyzer configuration and adapter contract.
Implementation requirements:
- Delete `PipelineConfig.Analyzer`.
- Delete `AnalyzerConfig` and `ArtifactSettings`.
- Remove analyzer timeout validation.
- Remove `stage.Env.Analyzer`.
- Delete `internal/adapters/analyzer` if no remaining code imports it.
- Remove `pipeline.analyzer.*` from tests, examples, and docs.
- Add or update strict-decode tests proving `pipeline.analyzer` is rejected.
Acceptance criteria:
- Analyze behavior remains fully Scriptorium-backed.
- No runtime code imports `internal/adapters/analyzer`.
- Pipeline configs containing `pipeline.analyzer` fail to load.
- Contributor and internal adapter docs no longer list the analyzer adapter.
### Stage 3: Remove Path-Based Previous Session Artifact Source
Status: Implemented
Remove `previous_session_artifact` and require canonical previous-session artifact sources.
Implementation requirements:
- Remove `previous_session_artifact` from supported Scriptorium input sources.
- Remove analyze-stage special-case handling that resolves `inputs.<name>.path` for previous artifacts.
- Keep canonical handling for `narratio.previous_session.artifact.<configured_artifact_key>`.
- Rewrite tests that use `previous_session_artifact` to use canonical sources and prepared previous-cache fixtures.
- Add validation tests proving `previous_session_artifact` is rejected.
- Update docs to remove the legacy path-based source and document only canonical previous-session sources.
Acceptance criteria:
- `pipeline.scriptorium.artifacts.*.inputs.*.source: previous_session_artifact` fails validation.
- Canonical previous-session sources continue to work for required and optional inputs.
- Prepare/restore previous-cache behavior remains unchanged.
- No docs or examples mention `previous_session_artifact` as supported.
## Test Guidance
Run focused tests after each stage:
- `go test ./internal/config -v`
- `go test ./internal/stage -run Analyze -v`
- `go test ./internal/app -v`
- `go test ./...`
For Stage 1, focus on config load/strict-decode and S3 workflow regression tests.
For Stage 2, focus on compile-time removal, config strict-decode tests, and full app/stage tests to catch stale adapter references.
For Stage 3, focus on Scriptorium config validation, analyze-stage input resolution, previous-cache behavior, and restore/analyze workflows.
## Documentation Updates
Update current-behavior docs only after the corresponding code removal lands:
- `docs/config.md`
- `docs/cli.md`, only if command behavior text references removed fields.
- `docs/operations.md`, only if operator workflow text references removed fields.
- `docs/internal/stage-analyze.md`
- `docs/internal/adapters.md`
- `examples/pipeline.full.annotated.yml`
- `examples/pipeline.production.yml`
Do not preserve removed fields in examples as compatibility notes. The goal is to make strict config behavior and documentation line up.
## Assumptions
- This is a hard cleanup; no backward-compatible aliases are retained.
- Current production configs can be migrated to `storage.s3.*`, Scriptorium artifacts, and canonical previous-session sources before this lands.
- Removing the unused analyzer adapter does not block any active stage behavior.
- The cleanup should be implemented in the listed order so inert schema removal is separated from behavior removal.

View File

@@ -1,255 +0,0 @@
# Roadmap: Session-Oriented CLI Cleanup
Status: Implemented
## Problem
Narratio's public CLI has accumulated too many top-level commands. Several
commands are session-scoped operator helpers, but they currently appear as
independent top-level verbs:
- `plan`
- `status`
- `restore`
- `artifacts list`
- `locks`
- `session validate`
- `session init`
This makes the command surface harder to learn because the CLI does not clearly
separate primary workflow actions from session inspection, initialization,
restore, and helper operations.
## Target Model
Keep primary workflow commands at top level:
- `run`
- `run-stage`
- `resume`
- `analyze`
- `publish`
- `clean`
- `session`
Keep `clean` top-level because it can operate on one session or all local
sessions and is a workspace maintenance command, not only a session helper.
Move session-scoped helper commands under `narratio session` and use positional
session identifiers:
- `narratio session init <session_id> [--remote|--output <path>] [--flags]`
- `narratio session validate <session_id> [--flags]`
- `narratio session status <session_id> [--flags]`
- `narratio session plan <session_id> [--flags]`
- `narratio session restore <session_id> [--flags]`
- `narratio session artifacts <session_id> [--remote] [--flags]`
- `narratio session locks <session_id> [--flags]`
- `narratio session locks add <session_id> <source> [--reason <text>] [--force] [--flags]`
- `narratio session locks remove <session_id> <source> [--flags]`
Update top-level workflow commands to use positional session identifiers:
- `narratio run <session_id> [--flags]`
- `narratio resume <session_id> [--flags]`
- `narratio analyze <session_id> [--flags]`
- `narratio publish <session_id> [--flags]`
- `narratio run-stage <stage> <session_id> [--flags]`
The positional session ID replaces `--session-id` as the primary public
interface. Existing `--config`, `--campaign`, `--session`, and
`--previous-session-id` flags remain available where they are meaningful.
## Command Mapping
| Current command | Target command |
| --- | --- |
| `narratio run --session-id <id>` | `narratio run <id>` |
| `narratio resume --session-id <id>` | `narratio resume <id>` |
| `narratio analyze --session-id <id>` | `narratio analyze <id>` |
| `narratio publish --session-id <id>` | `narratio publish <id>` |
| `narratio run-stage [flags] <stage> --session-id <id>` | `narratio run-stage <stage> <id> [flags]` |
| `narratio plan --session-id <id>` | `narratio session plan <id>` |
| `narratio status --session-id <id>` | `narratio session status <id>` |
| `narratio restore --session-id <id>` | `narratio session restore <id>` |
| `narratio artifacts list --session-id <id>` | `narratio session artifacts <id>` |
| `narratio locks --session-id <id>` | `narratio session locks <id>` |
| `narratio locks add --session-id <id> <source>` | `narratio session locks add <id> <source>` |
| `narratio locks remove --session-id <id> <source>` | `narratio session locks remove <id> <source>` |
| `narratio session validate --session-id <id>` | `narratio session validate <id>` |
| `narratio session init --session-id <id>` | `narratio session init <id>` |
| `narratio clean --session-id <id>` | `narratio clean <id>` |
| `narratio clean --all` | unchanged |
`clean` remains top-level, but its session-scoped form should also move from
`--session-id` to positional `<session_id>` for consistency.
## Compatibility Policy
This is a hard public CLI cleanup after the migration step lands.
During Step 1, old forms may remain as compatibility aliases to keep the
implementation reviewable. During Step 2, remove the old forms from command
dispatch, tests, docs, and examples:
- remove top-level `plan`;
- remove top-level `status`;
- remove top-level `restore`;
- remove top-level `artifacts`;
- remove top-level `locks`;
- remove `--session-id` from the public command syntax for session-aware
commands.
Do not keep long-term deprecated aliases unless a later roadmap explicitly
chooses a compatibility window.
`status --manifest` does not fit the session-oriented command shape. Remove it
from the public CLI in this cleanup. If direct manifest inspection is needed
later, add a separate diagnostic command in a future roadmap rather than keeping
it as a special case in `session status`.
## Implementation Step 1: Add New Session-Oriented Interface
Status: Implemented
Add the target command forms while preserving current behavior internally.
Implementation requirements:
- Add positional session ID parsing helpers in `internal/app`.
- Keep the existing `loadCommandConfig` behavior and populate
`config.SessionLoadOptions.SessionID` from the positional ID.
- Add or update command wrappers:
- `Run(ctx, args, out)` parses `run <session_id>`.
- `Resume(ctx, args, out)` parses `resume <session_id>`.
- `Analyze(ctx, args, out)` parses `analyze <session_id>`.
- `Publish(ctx, args, out)` parses `publish <session_id>`.
- `RunStage(ctx, args, out)` parses `run-stage <stage> <session_id>`.
- `Clean(ctx, args, out)` parses `clean <session_id>` and keeps
`clean --all`.
- Extend `Session(ctx, args, out)` dispatch to support:
- `init <session_id>`
- `validate <session_id>`
- `status <session_id>`
- `plan <session_id>`
- `restore <session_id>`
- `artifacts <session_id>`
- `locks <session_id>`
- `locks add <session_id> <source>`
- `locks remove <session_id> <source>`
- Keep storage access through the existing app-level object-store helper.
- Keep AWS SDK details behind storage adapters.
- Keep the runner, stages, manifest behavior, archive behavior, restore
planning, lock semantics, and artifact catalog behavior unchanged.
Acceptance criteria:
- New forms execute the same code paths and produce equivalent results.
- Positional session ID mismatch with concrete local or remote `session.yml`
fails through existing session identity checks.
- Remote session fallback still uses the positional session ID as the lookup
value.
- Current command tests cover the new forms before old forms are removed.
## Implementation Step 2: Remove Old Public Forms
Status: Implemented
Remove compatibility aliases and make the session-oriented interface the only
documented and supported public CLI.
Implementation requirements:
- Remove top-level dispatch for:
- `plan`
- `status`
- `restore`
- `artifacts`
- `locks`
- Remove `--session-id` flags from public session-aware commands.
- Keep `--previous-session-id` as an expected previous-session identity flag.
- Keep explicit `--session <path>` for loading a local concrete session file,
but still require the positional session ID for commands that operate on a
session.
- Remove `status --manifest`.
- Update usage text and invalid-command errors.
- Update `docs/cli.md` and `docs/operations.md` to use only the new forms.
- Update any roadmap docs that mention old helper command names.
- Update tests to expect old top-level helper commands and `--session-id` forms
to fail.
Acceptance criteria:
- Top-level command list is exactly:
- `run`
- `run-stage`
- `resume`
- `analyze`
- `publish`
- `clean`
- `session`
- All session-oriented commands use `narratio session <subcommand>
<session_id> [--flags]`, except nested lock mutation forms, which use
`narratio session locks add|remove <session_id> <source> [--flags]`.
- `clean <session_id>` and `clean --all` remain top-level.
- Current-behavior docs and tests no longer advertise `--session-id`.
## Test Guidance
Focused tests:
- `go test ./internal/app -run TestExecute -v`
- `go test ./internal/app -run 'Session|Status|Restore|Clean|Locks|Artifacts|Plan|RunStage|Analyze|Publish' -v`
- `go test ./internal/config -v`
Full validation:
- `go test ./...`
Test cases to add or update:
- `run <session_id>` loads local and remote sessions through the existing
config path.
- `resume <session_id>`, `analyze <session_id>`, and `publish <session_id>`
preserve current behavior.
- `run-stage <stage> <session_id>` preserves current run-stage output and
force/artifact-selection behavior.
- `session plan <session_id>` replaces top-level `plan`.
- `session status <session_id>` replaces top-level session status.
- `session validate <session_id>` replaces `session validate --session-id`.
- `session init <session_id>` writes the same local or remote concrete
`session.yml`.
- `session restore <session_id>` preserves restore planning/execution.
- `session artifacts <session_id> --remote` preserves promoted-output
availability reporting.
- `session locks <session_id>`, `session locks add <session_id> <source>`, and
`session locks remove <session_id> <source>` preserve static/remote lock
semantics.
- `clean <session_id>` preserves session cleanup behavior, while `clean --all`
remains unchanged.
- Old top-level helper commands fail after Step 2.
- `--session-id` fails after Step 2.
- `status --manifest` fails after Step 2.
## Documentation Guidance
Update only after implementation lands:
- `docs/cli.md`
- `docs/operations.md`
- any internal docs that list command names or examples
Keep planned behavior only in this roadmap until the command refactor is
implemented.
## Architecture Guardrails
- Keep Narratio explicit and stage-driven.
- Do not introduce a generic workflow or command framework abstraction.
- Reuse existing app command helpers where practical.
- Keep config loading strict and centralized.
- Keep storage details behind `storage.ObjectStore`.
- Keep secret-backed object-store construction in `internal/app`.
- Preserve manifest-driven resume and restore behavior.
- Treat command renaming as a public CLI contract change, not a runtime stage
behavior change.

View File

@@ -1,287 +0,0 @@
# Roadmap: Publish Contract
Status: Planned
## Problem
Narratio currently uses several terms for one operator-facing concept:
- `archive` is the stage that uploads run state and commits remote current
state.
- `publish` is the convenience command that force-runs the archive stage.
- `promote`, `promoted`, and `promote_artifacts` describe configured top-level
remote output writes.
This mixed vocabulary makes the public contract harder to explain. Operators
should not need to distinguish "archive the run", "publish the run", and
"promote artifacts" when these are all part of the same publish action.
The public model should use:
- `publish` for the stage, command, config section, and action;
- `published` for an expected remote output that exists at its top-level
current destination;
- `publish rules` for the configured source-to-destination output rules;
- `locked` for sources whose top-level published destination must not be
overwritten;
- `run history` for immutable per-run records under `runs/<run_id>/`.
## Target Model
The public stage is `publish`.
The convenience command:
narratio publish <session_id>
is equivalent to:
narratio run-stage publish <session_id> --force
Pipeline configuration uses `publish`:
publish:
enabled: true
upload_run: true
outputs:
- source: narratio.transcript.final_trimmed
- source: narratio.artifact.session_recap
locks:
- source: narratio.artifact.session_recap
reason: Final recap was manually edited.
Publish output rules are source-based. Each rule writes one artifact source to
a top-level remote destination. If `dest` is omitted, Narratio derives the
destination from the artifact registry or configured artifact output path.
The mutable remote lock store remains:
{session_prefix}/locks.yml
Remote availability output uses `published`:
Published:
- narratio.transcript.final_trimmed remote=published
- narratio.artifact.session_recap locked remote=published
The remote key layout is otherwise unchanged:
- immutable run history stays under `{session_prefix}/runs/{run_id}/`;
- current state stays under `{session_prefix}/current/manifest.json`;
- the final commit marker stays `{session_prefix}/current/run_id.txt`;
- `current/run_id.txt` is still written last.
## Compatibility Policy
This is a hard cutover.
After implementation:
- `pipeline.archive` is rejected by strict YAML decoding.
- `pipeline.archive.promote_artifacts` is rejected.
- `pipeline.workspace.cleanup_after_archive` is rejected.
- `pipeline.spool.delete_audio_after_archive` is rejected.
- `narratio run-stage archive <session_id>` is an unknown stage.
- manifests that record an `archive` stage are not migrated.
- old archive/promotion metadata keys are not read as compatibility fallbacks.
Existing remote objects are not moved or renamed. Remote layout remains stable;
the rename changes configuration, stage names, status output, metadata, helper
names, tests, examples, and documentation.
## Implementation Stages
### Stage 1: Public Schema and Stage Cutover
Status: Planned
Switch the public config and stage contract to publish terminology.
Implementation requirements:
- Replace `pipeline.archive` with `pipeline.publish`.
- Replace `archive.promote_artifacts` with `publish.outputs`.
- Keep output rule fields:
- `source`
- `dest`
- `required`
- Replace `pipeline.archive.locks` with `pipeline.publish.locks`.
- Rename post-publish cleanup fields:
- `pipeline.workspace.cleanup_after_publish`
- `pipeline.spool.delete_audio_after_publish`
- Rename the registered stage from `archive` to `publish`.
- Update stage order so `publish` runs after `analyze` and before `notify`.
- Update top-level `narratio publish` to target stage `publish`.
- Keep `run-stage --artifacts <names> publish` support.
- Reject `run-stage --artifacts <names>` for stages other than `analyze` and
`publish`.
- Preserve the remote commit ordering and storage adapter boundaries.
Acceptance criteria:
- `narratio run-stage publish <session_id>` executes the publish stage.
- `narratio publish <session_id>` force-runs the publish stage.
- `narratio run-stage archive <session_id>` fails clearly as an unknown stage.
- Old archive config fields fail strict decoding.
- New publish config fields load, default, and validate.
### Stage 2: Runtime Terminology and Metadata Cutover
Status: Planned
Rename implementation concepts and runtime output to publish terminology.
Implementation requirements:
- Rename archive/promotion config and runtime types conceptually to
publish/output terms.
- Rename the remote key helper intent from promoted artifact to published
output while keeping generated keys unchanged.
- Change helper output:
- `Promoted:` becomes `Published:`
- `remote=promoted` becomes `remote=published`
- lock output uses `published` / `not-published`
- Rename publish-stage metadata, including:
- `promoted_paths` to `published_paths`
- `promoted_files_uploaded` to `published_files_uploaded`
- `skipped_optional_promotions` to `skipped_optional_outputs`
- `skipped_unselected_promotions` to `skipped_unselected_outputs`
- `locked_promotion_count` to `locked_output_count`
- `locked_promotions` to `locked_outputs`
- Update previous-cache and restore logic to use the `publish` stage and
`published_paths` metadata only.
- Keep run-local stage output materialization separate from remote publish
terminology. If local helper names are confusing, rename them to
materialization-oriented names rather than publish names.
Acceptance criteria:
- Status and artifact helper output use `Published:` and `remote=published`.
- Publish metadata contains only publish/output terminology.
- Previous-cache and restore behavior works with publish metadata and does not
depend on old archive metadata.
- Storage adapters still receive explicit keys and no AWS SDK details leak into
app or stage logic.
### Stage 3: Documentation, Examples, and Final Cleanup
Status: Planned
Update implemented-behavior docs and remove stale public terminology after the
runtime cutover lands.
Implementation requirements:
- Update current-behavior docs:
- `docs/config.md`
- `docs/cli.md`
- `docs/operations.md`
- `docs/troubleshooting.md`
- `docs/architecture.md`
- relevant files under `docs/internal/`
- Rename `docs/internal/stage-archive.md` to
`docs/internal/stage-publish.md`.
- Update internal documentation links and references.
- Update examples to use:
- `publish.outputs`
- `publish.locks`
- `cleanup_after_publish`
- `delete_audio_after_publish`
- Update tests and final searches so old terminology remains only in this
roadmap as historical context.
Acceptance criteria:
- Maintained examples load and validate.
- Current-behavior docs describe only implemented publish terminology.
- Internal docs describe run history, published outputs, locks, and current
commit ordering clearly.
- Old user-facing archive/promote wording is removed except where discussing
historical behavior in this roadmap.
## Test Guidance
Focused tests:
- `go test ./internal/config -v`
- `go test ./internal/app -v`
- `go test ./internal/stage -v`
- `go test ./internal/artifacts -v`
Full validation:
- `go test ./...`
Config tests to add or update:
- `publish.outputs` defaults and validates.
- `publish.outputs[].dest` derives from the artifact registry when omitted.
- `publish.locks` validates with the same source rules as publish outputs.
- old `archive` fails strict decode.
- old `promote_artifacts` fails strict decode.
- old cleanup fields fail strict decode.
App and stage tests to add or update:
- stage order uses `publish` before `notify`.
- `run-stage publish` succeeds.
- `run-stage archive` fails clearly.
- `narratio publish` force-runs the `publish` stage.
- `--artifacts` is accepted for `run-stage publish`.
- `--artifacts` error text names `analyze` and `publish`.
- status and artifact list output show `Published:` and `remote=published`.
- lock output says `published` or `not-published`.
- previous-cache and restore use `publish` stage metadata.
Final searches:
- Config/stage names:
- `pipeline.archive`
- `archive:`
- `promote_artifacts`
- `cleanup_after_archive`
- `delete_audio_after_archive`
- User-facing output:
- `Promoted:`
- `remote=promoted`
- `not-promoted`
- Runtime symbols and metadata:
- `ArchiveConfig`
- `ArchivePromotionRule`
- `S3PromotedArtifactKey`
- `promoted_paths`
- `promoted_files_uploaded`
- `locked_promotions`
Expected remaining matches should be limited to this roadmap and narrowly
justified historical references until the roadmap is fully retired.
## Architecture Guardrails
- Keep Narratio explicit and stage-driven.
- Do not introduce a generic workflow or DAG abstraction.
- Keep strict YAML decoding.
- Keep remote path construction centralized.
- Keep storage details behind `storage.ObjectStore`.
- Keep AWS SDK types inside storage adapters.
- Preserve manifest-driven resume and restore behavior.
- Preserve current-state commit ordering with `current/run_id.txt` written
last.
- Keep raw secrets out of configs, manifests, logs, generated configs, and
publish metadata.
- Keep planned behavior only in this roadmap until implementation lands.
## Assumptions
- This is a breaking public/config/stage contract change.
- No compatibility aliases are retained.
- No migration logic is needed for in-progress local manifests.
- No migration logic is needed for old remote manifests.
- Existing remote objects are not moved or renamed.
- `publish` means uploading run history, writing configured published outputs,
and committing current state.
- `run history` is the preferred term for immutable per-run records under
`runs/<run_id>/`.
- `archive` remains acceptable only as a generic English concept in historical
roadmap context, not as a public Narratio command, config field, stage name,
or metadata term after implementation.

View File

@@ -1,210 +0,0 @@
# Roadmap: Transcript Artifact Naming
Status: Implemented
## Problem
Narratio's built-in transcript artifact names and canonical paths currently mix
operator-facing artifact meaning with historical stage and tool terminology:
- `narratio.transcript.merged` maps to `transcripts/merged.json`.
- `narratio.transcript.polished` maps to `transcripts/processed.json`.
- `narratio.transcript.full` maps to `transcripts/normalized.json`.
- `narratio.transcript.trimmed` maps to `transcripts/trimmed.json`.
This makes the public artifact surface harder to reason about. Operators see
`full`, `normalized`, `processed`, `polished`, `merged`, and `trimmed` used in
different places for the same transcript lineage.
The transcript source IDs, canonical paths, and manifest output kinds should
use one vocabulary based on each transcript's role in the session artifact
model.
## Target Model
Built-in transcript artifacts should use these public source IDs, canonical
paths, and manifest output kinds:
| Source ID | Canonical path | Output kind | Meaning |
| --- | --- | --- | --- |
| `narratio.transcript.base` | `transcripts/base.json` | `transcript_base` | First unified transcript produced by merging per-speaker raw transcripts. |
| `narratio.transcript.polished` | `transcripts/polished.json` | `transcript_polished` | Audita-polished transcript. |
| `narratio.transcript.final` | `transcripts/final.json` | `transcript_final` | Full final transcript after normalization. |
| `narratio.transcript.final_trimmed` | `transcripts/final.trimmed.json` | `transcript_final_trimmed` | Trimmed version of the final transcript. |
Stage names remain process-oriented and unchanged:
- `merge`
- `polish`
- `normalize`
- `trim`
Downstream adapter contracts also remain process-oriented. The rename changes
Narratio's artifact model, canonical paths, config examples, archive promotion
sources, lock sources, status output, and documentation. It should not rename
the stages themselves or move external integration details into stage logic.
## Compatibility Policy
This is a hard cutover.
After implementation, these old source IDs should be rejected:
- `narratio.transcript.merged`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
These old canonical paths should not be compatibility fallbacks:
- `transcripts/merged.json`
- `transcripts/processed.json`
- `transcripts/normalized.json`
- `transcripts/trimmed.json`
Existing remote archives are not migrated automatically. Operators who want
new promoted keys for old sessions should republish those sessions after
updating configuration.
## Implementation Stages
### Stage 1: Centralize Transcript Artifact Naming
Status: Implemented
Consolidate transcript artifact source IDs, canonical paths, and output kinds
in the artifact/path layer before changing runtime behavior.
Implementation requirements:
- Add or consolidate constants/helpers for built-in transcript source IDs.
- Add or consolidate constants/helpers for canonical transcript paths.
- Add or consolidate constants/helpers for transcript manifest output kinds.
- Keep source ID, path, and output-kind mappings in one registry or one
obviously shared artifact model.
- Update artifact registry tests to prove the target mapping.
- Avoid changing stage output behavior in this stage unless the implementation
is simpler and still reviewable.
Acceptance criteria:
- There is one clear source of truth for built-in transcript artifact names,
paths, and output kinds.
- Tests prove the new target mapping in the artifact layer.
- No generic workflow abstraction is introduced.
### Stage 2: Rename Runtime Outputs and Defaults
Status: Implemented
Switch runtime behavior to the new transcript artifact model.
Implementation requirements:
- Update `merge` to write and record `transcripts/base.json` with
`transcript_base`.
- Update `polish` to write and record `transcripts/polished.json` with
`transcript_polished`.
- Update `normalize` to write and record `transcripts/final.json` with
`transcript_final`.
- Update `trim` to write and record `transcripts/final.trimmed.json` with
`transcript_final_trimmed`.
- Update normalize and trim defaults to:
- `pipeline.normalize.output_path: transcripts/final.json`
- `pipeline.trim.output_path: transcripts/final.trimmed.json`
- Update built-in artifact resolution, archive promotion destination
derivation, archive locks, status output, artifact catalog output,
previous-cache resolution, restore planning, and restore execution to use
the new registry values.
- Ensure old source IDs fail config validation.
Acceptance criteria:
- New runs produce the target canonical transcript files.
- Manifest outputs use the target output kinds.
- Archive promotion and lock validation accept new source IDs and reject old
source IDs.
- Status and artifact listing display new source IDs.
- Restore uses the new canonical paths and does not restore old transcript
paths as canonical outputs.
### Stage 3: Update Tests, Examples, and Current Documentation
Status: Implemented
Update all implemented-behavior references after the runtime cutover lands.
Implementation requirements:
- Update examples to use `narratio.transcript.final_trimmed` and
`transcripts/final.trimmed.json` where trimmed final transcript is intended.
- Update examples that refer to full final transcripts to use
`narratio.transcript.final` and `transcripts/final.json`.
- Update `docs/config.md`, `docs/internal/artifacts.md`, stage docs,
CLI examples, operations examples, archive examples, lock examples, and
status/artifact-list examples.
- Add strict validation tests proving old source IDs are rejected.
- Mark roadmap stages implemented only after code, tests, examples, and
current-behavior docs agree.
Acceptance criteria:
- Maintained examples load and validate.
- Current-behavior docs describe only implemented new names.
- Old names remain only in this roadmap as historical/planning context until
this roadmap is retired or archived.
## Test Guidance
Run focused tests while implementing:
- `go test ./internal/artifacts -v`
- `go test ./internal/config -v`
- `go test ./internal/stage -v`
- `go test ./internal/app -v`
Run full validation before finishing:
- `go test ./...`
Run final searches:
- Old source IDs:
- `narratio.transcript.merged`
- `narratio.transcript.full`
- `narratio.transcript.trimmed`
- Old paths:
- `transcripts/merged.json`
- `transcripts/processed.json`
- `transcripts/normalized.json`
- `transcripts/trimmed.json`
- Old output kinds:
- `transcript_merged`
- `transcript_processed`
- `transcript_normalized`
- `transcript_trimmed`
Expected remaining matches should be limited to this roadmap's
historical/planning references until the roadmap is fully completed.
## Architecture Guardrails
- Keep Narratio explicit and stage-driven; do not introduce a generic workflow
or DAG abstraction.
- Keep path and artifact naming in centralized helpers rather than scattered
string concatenation.
- Preserve manifest-driven resume behavior.
- Keep storage details behind storage adapters.
- Do not move Seriatim, Audita, or Scriptorium command details out of their
adapter boundaries.
- Keep current-behavior documentation in sync only after implementation lands;
planned behavior belongs in this roadmap until then.
## Assumptions
- The cutover is intentionally not backward-compatible.
- Existing remote archive objects are not renamed or migrated automatically.
- Stage names and downstream adapter request field names remain unchanged.
- The term `base` is preferred over `merged` for the first unified transcript.
- The term `final` is preferred over `full` or `normalized` for the full final
transcript.
- The trimmed final path is `transcripts/final.trimmed.json`.

View File

@@ -1,35 +1,62 @@
# Troubleshooting
## Purpose
Canonical operator troubleshooting guide for recurring Narratio failures.
Operational diagnosis guide for common Narratio failures.
## Config discovery failure
## Config file not found
Symptom:
- command fails because `pipeline.yml`, `campaign.yml`, or `session.yml` was not found.
Likely cause:
- missing files in discovery paths.
- missing/incorrect campaign selection.
- local file exists but was not passed explicitly.
- command fails to resolve `pipeline.yml`, `campaign.yml`, or `session.yml`.
Likely causes:
- missing files in default search paths;
- wrong campaign selection;
- omitted explicit flags.
Diagnostics:
```bash
ls -l /usr/local/etc/narratio/pipeline.yml /etc/narratio/pipeline.yml
ls -l /usr/local/etc/narratio/session.yml /etc/narratio/session.yml
narratio session plan 2026-04-04
```
Safe fix:
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
## Templated session file rejected
Relevant reference: [Configuration discovery](./config.md#discovery-and-selection).
## Session template placeholders rejected
Symptom:
- load fails because `session.yml` must be concrete.
- load error says session file must be concrete or contains `{{ ... }}` placeholders.
Likely cause:
- template placeholders (`{{ ... }}`) still present in loaded session config.
- using template content as runtime session config.
Diagnostics:
```bash
narratio session validate 2026-04-04 --session /path/session.yml
```
Safe fix:
- generate concrete session YAML with `narratio session init`.
Relevant reference: [Operations: Session Initialization](./operations.md#session-initialization).
## Strict decode or schema validation failure
Symptom:
- unknown field / invalid value error during config load.
Likely cause:
- stale field name, typo, invalid enum, or invalid duration/path format.
Diagnostics:
@@ -38,82 +65,241 @@ narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /pa
```
Safe fix:
- generate concrete session YAML via `narratio session init`.
## Strict decode or validation failure
- align config with [Configuration](./config.md) and the
[maintained examples](../examples/README.md).
Relevant reference: [Configuration](./config.md).
## Audio mode conflict
Symptom:
- unknown field or invalid value error during config load.
- validation fails on session audio configuration.
Likely cause:
- typo, stale field name, or invalid value.
- configured both local and S3 session audio inputs.
Diagnostics:
```bash
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
narratio session validate 2026-04-04
```
Safe fix:
- align config with [docs/config.md](./config.md) and maintained examples.
## `--artifacts` selection failure
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
Relevant reference: [Session configuration](./config.md#session).
## `--artifacts` selection error
Symptom:
- command fails on unknown/invalid selected artifact key.
Likely cause:
- artifact key not defined in `pipeline.scriptorium.artifacts`.
- empty token in `--artifacts` input.
- unknown artifact key or invalid `--artifacts` usage.
Safe fix:
- use only configured artifact keys.
Likely causes:
## `run-stage --artifacts` unsupported stage
Symptom:
- `run-stage` rejects `--artifacts` for the selected stage.
Likely cause:
- `--artifacts` used with a stage other than `analyze` or `publish`.
Safe fix:
- use `--artifacts` only with `run-stage analyze ...` or `run-stage publish ...`.
## Previous-session input unavailable
Symptom:
- analyze fails on required previous-session artifact input.
Likely cause:
- `previous/**` cache not hydrated for this session.
- key not defined in `pipeline.scriptorium.artifacts`;
- empty list entry (for example trailing comma);
- `run-stage` used with non-`analyze`/`publish` target.
Diagnostics:
```bash
narratio session artifacts 2026-04-04
```
Safe fix:
- provide only configured keys and use `--artifacts` with supported commands/stages.
Relevant reference: [CLI artifact selection](./cli.md).
## Notarius executable missing
Symptom:
- extraction fails while resolving or starting the Notarius executable.
Likely causes:
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
- a configured executable path is wrong.
Safe fix:
- install a compatible Notarius release or correct the binary setting, then
rerun extraction.
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
and [Notarius integration](./integrations/notarius.md).
## Notarius exits nonzero
Symptom:
- extraction reports a Notarius exit error instead of a receipt.
Diagnostics:
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
the receipt and is not merged with diagnostics.
Safe fix:
- correct the reported Notarius pipeline, input, provider, or configuration
failure and rerun extraction. Do not edit a staged output bundle into place.
After a failed replacement, an older immutable bundle may still exist even
though the current session manifest has no successful extraction payload. This
is expected audit state, not a signal to relink the old bundle manually.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Atomic Notarius promotion unsupported
Symptom:
- extraction fails with `atomic no-replace directory promotion is unsupported`
before a durable bundle or temporary promotion tree is created.
Likely cause:
- Narratio is running on an operating system other than Linux, macOS, or
Windows, where the required atomic no-replace directory primitive has not
been implemented and verified.
Safe fix:
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
with a manual copy or move; the session manifest must never observe a partial
or overwritten bundle.
This is an extraction-specific platform boundary, not a support statement for
unrelated Narratio workflows. See
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Notarius receipt or index incompatible
Symptom:
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
lane descriptor, or payload path even though Notarius exited successfully.
Likely causes:
- Narratio and Notarius versions disagree on their consumer contract;
- the configured pipeline or lane constraints are stale;
- output paths escape the bundle or traverse symlinks.
Safe fix:
- compare installed Notarius output with the canonical Notarius contracts,
including receipt `index_file: index.json` and index management names
`manifest.json`, `rejected.json`, and `warnings.json`; align
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
checks.
Relevant reference: [Notarius integration](./integrations/notarius.md).
## Required Notarius lane rejected or missing
Symptom:
- extraction fails because a configured lane is rejected, missing, duplicated,
or incompatible, including after a zero exit.
Safe fix:
- inspect the Notarius diagnostic log and bundle rejection/warning information;
- correct the Notarius module or the exact declared lane contract;
- remove an output declaration only if downstream consumers genuinely no longer
require that source, then rerun extraction.
Every configured output is required. Narratio does not promote a partial result.
## Extraction resume invalidated
Symptom:
- a previously successful extraction runs again during ordinary continuation.
Likely causes:
- the executable/config path, pipeline ID, timeout, working directory, or
configured output contracts changed;
- the durable bundle, index, lane set, provenance, regular-file status, or
checksum no longer validates.
Safe fix:
- allow the automatic rerun after verifying the current configuration. Treat
an unsafe path or symlink error as filesystem corruption or tampering and
investigate it rather than replacing files manually.
## Notarius transitive configuration changed
Symptom:
- Notarius profiles, prompts, modules, imported files, or references changed,
but Narratio still considers the previous extraction resumable.
Safe fix:
```bash
narratio run-stage extract 2026-04-04 --force
```
Narratio fingerprints its invocation contract, not the contents of transitive
Notarius inputs. Always force extraction after changing them; downstream
successful stages are then marked stale normally.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Previous-session artifact input missing
Symptom:
- prepare/analyze fails due to missing required previous-session artifact cache input.
Likely causes:
- missing `session.previous_session_id`;
- previous artifact not restored/published for source session.
Diagnostics:
```bash
narratio session validate 2026-04-04
narratio session status 2026-04-04
```
Safe fix:
```bash
narratio session restore 2026-04-04
```
or rerun prepare after correcting session config:
```bash
narratio run-stage prepare 2026-04-04 --force
```
Or rehydrate from remote current state:
```bash
narratio session restore 2026-04-04
```
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
## Session lock conflict (`.lock`)
Symptom:
- command fails with lock conflict.
Likely cause:
- another process is running for the same session.
- stale lock file from interrupted command.
- command fails acquiring session lock.
Likely causes:
- another process is running for the same session;
- a process still holds the operating-system lock while it is shutting down.
Diagnostics:
@@ -123,16 +309,22 @@ ps aux | grep narratio
```
Safe fix:
- wait for active process; remove stale lock only if no process is active.
## Restore current pointer/manifest missing
- wait for active process completion;
- retry after an interrupted holder has exited; the kernel releases its lock
even though the `.lock` metadata file remains for inspection.
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
## Restore conflict without `--force`
Symptom:
- restore fails reading remote current state.
- restore fails with conflict count.
Likely cause:
- publish commit did not complete.
- `current/run_id.txt` or `current/manifest.json` is missing.
- local durable files differ from remote restore sources.
Diagnostics:
@@ -141,83 +333,148 @@ narratio session restore 2026-04-04 --dry-run
```
Safe fix:
- republish from a healthy local session state.
## Restore conflict without `--force`
- review conflicts;
- rerun with `--force` only when remote state should overwrite local.
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
## Restore current-state discovery failure
Symptom:
- restore reports conflict and exits.
Likely cause:
- local durable file differs from remote restore source.
- restore cannot find current pointer or current manifest.
Safe fix:
- inspect with `--dry-run`.
- rerun with `--force` only when remote should overwrite local.
Likely causes:
## Secrets or credentials failure
Symptom:
- startup fails loading secrets dir, or storage/tool auth fails at runtime.
Likely cause:
- invalid `pipeline.secrets.env_dir`.
- missing credential env vars.
Diagnostics:
```bash
ls -la /path/to/secrets_dir
env | grep -E 'AUDITA|OBJECT_STORAGE|AWS|SCRIPTORIUM'
```
Safe fix:
- fix path/permissions/env vars; keep secret values out of YAML.
## S3 audio prepare failure
Symptom:
- prepare fails in S3 mode (list/download/no files/backend error).
Likely cause:
- bad `session.inputs.audio_s3.prefix`.
- no `.flac` objects at prefix.
- bad storage credentials/config.
- mixed local+S3 audio config.
Diagnostics:
```bash
narratio run-stage prepare 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
```
Safe fix:
- configure exactly one audio mode and verify storage access.
## Publish output or current-pointer failure
Symptom:
- publish fails on required output source missing, upload error, or commit-marker write failure.
Likely cause:
- required source file not produced.
- storage upload failed before `current/run_id.txt` write.
- no committed publish current state;
- storage credentials or connectivity failure.
Diagnostics:
```bash
narratio session status 2026-04-04
narratio run-stage publish 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
narratio session restore 2026-04-04 --dry-run
```
Safe fix:
- rerun upstream stages to regenerate required outputs.
- adjust `pipeline.publish.outputs` source/dest rules.
- retry after storage issue is fixed.
## Helpful Links
- resolve storage/auth issue;
- republish from healthy local state if current pointer is missing.
Relevant reference: [Operations: Publish Workflow](./operations.md#publish-workflow).
## Publish output failure
Symptom:
- publish fails on missing required source, upload error, or commit write.
Likely causes:
- required source file not produced;
- lock/state expectations mismatch;
- remote storage failure.
Diagnostics:
```bash
narratio session artifacts 2026-04-04 --remote
narratio session status 2026-04-04
narratio run-stage publish 2026-04-04 --force
```
Safe fix:
- regenerate missing sources by rerunning prerequisite stages;
- correct publish source/destination rules;
- retry after storage failure is resolved.
Relevant reference: [Publish configuration](./config.md#publish-configuration-summary).
## Render markdown source missing
Symptom:
- analyze or publish fails because `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` is unavailable.
Likely causes:
- render stage was not executed after transcript changes;
- render stage failed before producing canonical markdown outputs.
Diagnostics:
```bash
narratio session status 2026-04-04
```
Safe fix:
- rerun render and then retry downstream stage(s):
```bash
narratio run-stage render 2026-04-04 --force
narratio run-stage analyze 2026-04-04 --force
```
Relevant reference: [Operations: Stage Execution](./operations.md#stage-execution-and-continuation-behavior).
## Secrets or storage credential failure
Symptom:
- object-store command fails at initialization/auth.
Likely causes:
- invalid `pipeline.secrets.env_dir`;
- missing credential environment variables;
- invalid S3 endpoint/bucket settings.
Diagnostics:
```bash
ls -la /path/to/secrets_dir
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
```
Safe fix:
- correct secret-file path and permissions;
- provide required env vars;
- keep secret values out of YAML.
Relevant reference: [Secrets](./config.md#secrets-handling).
## S3 audio prepare failure
Symptom:
- prepare fails listing/downloading session S3 audio.
Likely causes:
- incorrect `session.inputs.audio_s3.prefix`;
- no matching `.flac` objects;
- storage connectivity or permissions failure.
Diagnostics:
```bash
narratio session validate 2026-04-04
```
Safe fix:
- verify prefix contents and storage access;
- keep session audio mode consistent.
Relevant reference: [Operations](./operations.md).
## References
- [docs/config.md](./config.md)
- [docs/cli.md](./cli.md)
- [docs/config.md](./config.md)
- [docs/operations.md](./operations.md)
- [docs/internal/stage-publish.md](./internal/stage-publish.md)

48
examples/README.md Normal file
View File

@@ -0,0 +1,48 @@
# Maintained Examples
These files are safe, copyable starting points for Narratio configuration and
input structure. Replace placeholder identifiers, storage names, integration
URLs, and paths for the target environment. Field meanings and defaults belong
in the [configuration reference](../docs/config.md).
## Pipeline Configuration
- [Minimal pipeline](pipeline.minimal.yml): campaign discovery plus the required
WhisperX URL.
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
external tools, and configured Scriptorium artifacts.
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
pipeline section with explanatory comments.
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
Scriptorium artifact consuming only three declared Notarius lanes.
The existing `internal/config` example test loads and validates each pipeline
with the sample campaign and a compatible local- or S3-audio session.
## Campaign And Session Configuration
- [Sample campaign](campaigns/sample-campaign/campaign.yml), its
[session template](campaigns/sample-campaign/session.template.yml), and its
adjacent stable inputs provide a complete campaign directory shape.
- [Local-audio session](session.local-audio.yml) and
[S3-audio session](session.s3-audio.yml) are concrete session files.
- [Session template](session.template.yml) and the campaign-local equivalent
demonstrate the narrow placeholder syntax consumed by `session init`; they
are templates, not runtime session files.
## Input Fixtures
- [Speakers](speakers.yml), [autocorrect](autocorrect.yml), and
[glossary](glossary.yml) show the standalone input shapes.
- The sample campaign references its local
[speakers](campaigns/sample-campaign/speakers.yml),
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
[glossary](campaigns/sample-campaign/glossary.yml),
[players](campaigns/sample-campaign/players.yml), and
[party](campaigns/sample-campaign/party.yml) fixtures.
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
reserves the expected filename and directory shape. Replace it with a real
FLAC file before running transcription.
The examples contain environment-variable names but no credential values. They
use fictional campaign content and reserved example domains.

View File

@@ -4,3 +4,5 @@ inputs:
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
players_file: ./players.yml
party_file: ./party.yml

View File

@@ -0,0 +1,2 @@
- name: Example Hero
type: pc

View File

@@ -0,0 +1,2 @@
- name: Example Player
role: player

View File

@@ -1,5 +1,5 @@
match:
- speaker: "Eric Rakestraw"
- speaker: "Example Speaker"
match:
- "Eric_Rakestraw"
- "Eric"
- "Example_Speaker"
- "Example"

View File

@@ -0,0 +1,55 @@
# Purpose-specific extraction example: a Scriptorium session brief consumes
# only the three Notarius lanes it needs.
campaigns:
root: /usr/local/share/narratio/campaigns
default_campaign_id: sample-campaign
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
notarius:
enabled: true
binary: notarius
config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session
timeout: 3h
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
location_registry:
lane_id: location-registry
media_type: application/json
schema_id: notarius.dnd.location_registry
schema_version: v1
module_key: dnd/location-registry
scene_descriptions:
lane_id: scene-descriptions
media_type: application/json
schema_id: notarius.dnd.scene_descriptions
schema_version: v1
module_key: dnd/scene-descriptions
scriptorium:
binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml
artifacts:
session_brief:
enabled: true
prompt_id: dnd.session_brief
output_path: artifacts/session_brief.md
inputs:
npcs:
source: narratio.extraction.npc_registry
required: true
locations:
source: narratio.extraction.location_registry
required: true
scenes:
source: narratio.extraction.scene_descriptions
required: true

View File

@@ -12,7 +12,7 @@ workspace:
# env_dir: ./secrets
storage:
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
backend: s3
s3:
# Required when using S3 audio or S3 publish uploads.
@@ -48,12 +48,23 @@ publish:
- source: narratio.transcript.final_trimmed
dest: transcripts/final.trimmed.json
required: true
- source: narratio.transcript.final_markdown
dest: transcripts/final.md
required: true
- source: narratio.transcript.final_trimmed_markdown
dest: transcripts/final.trimmed.md
required: true
- source: narratio.artifact.session_recap
dest: artifacts/session_recap.md
required: true
- source: narratio.artifact.player_handout
dest: artifacts/player_handout.md
required: false
# Extraction lanes publish only when named explicitly; the bundle and index
# are never implicit publish sources.
- source: narratio.extraction.npc_registry
dest: artifacts/extraction/npc-registry.json
required: true
whisperx:
# Required.
@@ -104,20 +115,91 @@ normalize:
report: true
trim:
# Keep disabled unless bounds prompt integration is configured.
enabled: false
# Optional; defaults shown explicitly.
enabled: true
output_path: transcripts/final.trimmed.json
bounds:
prompt_id: dnd.session_bounds
profile_id: local-fast
profile_id: ""
transcript_input_name: transcript
output_path: reports/session_bounds.json
output_path: artifacts/session_bounds.json
timeout: 10m
render_debug: false
render_output_path: reports/session_bounds.render.json
seriatim:
report: false
notarius:
# Optional structured extraction between trim and render.
enabled: true
binary: notarius
config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session
timeout: 3h
working_directory: /usr/local/etc/notarius
# Each key creates source narratio.extraction.<key>. These constraints match
# the current Notarius D&D lane contracts; update them with Notarius.
outputs:
item_registry:
lane_id: item-registry
media_type: application/json
schema_id: notarius.dnd.item_registry
schema_version: v1
module_key: dnd/item-registry
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
location_registry:
lane_id: location-registry
media_type: application/json
schema_id: notarius.dnd.location_registry
schema_version: v1
module_key: dnd/location-registry
scene_descriptions:
lane_id: scene-descriptions
media_type: application/json
schema_id: notarius.dnd.scene_descriptions
schema_version: v1
module_key: dnd/scene-descriptions
item_occurrences:
lane_id: item-occurrences
media_type: application/json
schema_id: notarius.dnd.item_occurrences
schema_version: v1
module_key: dnd/item-occurrences
spells:
lane_id: spells
media_type: application/json
schema_id: notarius.dnd.spells
schema_version: v1
module_key: dnd/spells
combat_turns:
lane_id: combat-turns
media_type: application/json
schema_id: notarius.dnd.combat_turns
schema_version: v1
module_key: dnd/combat-turns
npc_occurrences:
lane_id: npc-occurrences
media_type: application/json
schema_id: notarius.dnd.npc_occurrences
schema_version: v1
module_key: dnd/npc-occurrences
location_occurrences:
lane_id: location-occurrences
media_type: application/json
schema_id: notarius.dnd.location_occurrences
schema_version: v1
module_key: dnd/location-occurrences
enemy_events:
lane_id: enemy-events
media_type: application/json
schema_id: notarius.dnd.enemy_events
schema_version: v1
module_key: dnd/enemy-events
scriptorium:
binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml
@@ -138,6 +220,15 @@ scriptorium:
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
players:
source: narratio.input.players
required: true
party:
source: narratio.input.party
required: true
glossary:
source: narratio.input.glossary
required: false
vars:
session_id: true
session_date: true
@@ -169,7 +260,5 @@ scriptorium:
output_kind: player_handout
notification:
# Optional notification settings.
backend: ""
recipient: ""
timeout: 30s
# No delivery provider is currently implemented.
mode: noop

View File

@@ -26,6 +26,12 @@ publish:
- source: narratio.transcript.final_trimmed
dest: transcripts/final.trimmed.json
required: true
- source: narratio.transcript.final_markdown
dest: transcripts/final.md
required: true
- source: narratio.transcript.final_trimmed_markdown
dest: transcripts/final.trimmed.md
required: true
- source: narratio.artifact.session_recap
dest: artifacts/session_recap.md
required: true
@@ -65,9 +71,6 @@ normalize:
output_schema: seriatim-intermediate
report: true
trim:
enabled: false
scriptorium:
binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml
@@ -87,6 +90,15 @@ scriptorium:
previous_recap:
source: narratio.previous_session.artifact.session_recap
required: false
players:
source: narratio.input.players
required: true
party:
source: narratio.input.party
required: true
glossary:
source: narratio.input.glossary
required: false
vars:
session_id: true
session_date: true
@@ -113,4 +125,4 @@ scriptorium:
output_kind: player_handout
notification:
timeout: 30s
mode: noop

View File

@@ -1,5 +1,5 @@
match:
- speaker: "Eric Rakestraw"
- speaker: "Example Speaker"
match:
- "Eric_Rakestraw"
- "Eric"
- "Example_Speaker"
- "Example"

1
go.mod
View File

@@ -7,6 +7,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/smithy-go v1.25.1
golang.org/x/sys v0.47.0
gopkg.in/yaml.v3 v3.0.1
)

2
go.sum
View File

@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -4,10 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// NoopRunner is a deterministic no-op audita adapter.
@@ -84,17 +84,17 @@ func materializePlaceholders(req PolishRequest) error {
"merged_transcript_path": req.MergedTranscriptPath,
"output_path": req.OutputProcessedPath,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
@@ -117,14 +117,14 @@ func writeJSONIfRequested(path string, payload any) error {
if path == "" {
return nil
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
}
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
}
if err := subprocess.WriteFileAtomic(path, data, 0o644); err != nil {
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write placeholder json %q: %w", path, err)
}
return nil

View File

@@ -6,8 +6,6 @@ import (
"time"
)
// TODO: implement a real Audita subprocess/service adapter.
// Runner is the adapter boundary for audita polish invocations.
type Runner interface {
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
@@ -22,16 +20,6 @@ type PolishRequest struct {
ReportPath string
WorkDir string
Modules []string
BaseURL string
Model string
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
StdoutLogPath string
StderrLogPath string
}

View File

@@ -11,8 +11,15 @@ import (
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
// MaxReportOutputBytes bounds Audita's optional report JSON result.
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
type SubprocessRunnerConfig struct {
Binary string
@@ -211,6 +218,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
Args: args,
Timeout: r.timeout,
EnvOverrides: env,
DiagnosticOwner: "audita",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -370,13 +378,13 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
}
func validateProcessedOutput(path string) error {
data, err := os.ReadFile(path)
data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
if err != nil {
return fmt.Errorf("read file: %w", err)
return err
}
var payload map[string]any
@@ -405,9 +413,9 @@ func addSubprocessStreamHint(message string, runErr error) string {
}
func validateJSONFile(path string) error {
data, err := os.ReadFile(path)
data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
if err != nil {
return fmt.Errorf("read file: %w", err)
return err
}
var v any
if err := json.Unmarshal(data, &v); err != nil {
@@ -415,3 +423,11 @@ func validateJSONFile(path string) error {
}
return nil
}
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
data, err := fileops.ReadRegularFile(path, limit)
if err != nil {
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
}
return data, nil
}

View File

@@ -189,7 +189,7 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
}
}
func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
@@ -214,8 +214,8 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
}
rec := readAuditaHelperRecord(t, recordPath)
if rec.Env["AUDITA_INHERITED_MARKER"] != "inherited-from-parent" {
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want inherited-from-parent", rec.Env["AUDITA_INHERITED_MARKER"])
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
}
}

View File

@@ -0,0 +1,22 @@
package notarius
import "context"
// FakeRunner is a configurable in-memory runner for stage tests.
type FakeRunner struct {
Requests []RunRequest
Result RunResult
Err error
}
// Run records the request and returns the configured result or error.
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
if err := ctx.Err(); err != nil {
return RunResult{}, err
}
f.Requests = append(f.Requests, req)
if f.Err != nil {
return RunResult{}, f.Err
}
return f.Result, nil
}

View File

@@ -0,0 +1,108 @@
// Package notarius declares the adapter contract for Notarius CLI invocations.
package notarius
import (
"context"
"time"
)
const ReceiptSchemaVersion = "notarius.run-result.v1"
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
type Runner interface {
Run(ctx context.Context, req RunRequest) (RunResult, error)
}
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
type RunRequest struct {
Binary string
ConfigPath string
PipelineID string
InputPath string
OutputRoot string
WorkingDirectory string
ReceiptPath string
LogPath string
Timeout time.Duration
}
// Receipt is the transport-neutral successful run receipt.
type Receipt struct {
SchemaVersion string
RunID string
PipelineID string
OutputDirectory string
IndexFile string
NormalizedOutputCount int
RejectedOutputCount int
WarningCount int
ValidationStatus string
DebugDirectory string
}
// LaneDescriptor identifies one normalized lane payload discovered through the index.
type LaneDescriptor struct {
LaneID string
File string
Path string
MediaType string
ModuleKey string
SchemaID string
SchemaName string
SchemaVersion string
}
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
type PipelineDescriptor struct {
ArtifactKind string
File string
Path string
MediaType string
SchemaID string
SchemaName string
SchemaVersion string
}
// Index describes the validated bundle-management and artifact paths.
type Index struct {
Path string
ManifestFile string
ManifestPath string
RejectedFile string
RejectedPath string
WarningsFile string
WarningsPath string
Lanes []LaneDescriptor
ChunkMap *PipelineDescriptor
EvidenceContext *PipelineDescriptor
}
// RejectionSummary retains structured rejection identity without free-form messages.
type RejectionSummary struct {
Stage string
StepID string
LaneID string
ModuleKey string
ChunkID string
ValidatorName string
ReasonCode string
}
// WarningSummary retains structured warning identity without free-form messages.
type WarningSummary struct {
Scope string
ReasonCode string
}
// RunResult describes a successfully decoded and validated Notarius bundle.
type RunResult struct {
Receipt Receipt
Index Index
BundleRoot string
ReceiptPath string
LogPath string
ExitCode int
Duration time.Duration
Rejections []RejectionSummary
Warnings []WarningSummary
}

View File

@@ -0,0 +1,502 @@
package notarius
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
const (
maxReceiptBytes = 1 << 20
maxIndexBytes = 4 << 20
maxSummaryBytes = 4 << 20
canonicalIndexFile = "index.json"
canonicalManifestFile = "manifest.json"
canonicalRejectedFile = "rejected.json"
canonicalWarningsFile = "warnings.json"
)
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
// SubprocessRunner invokes Notarius through its public CLI.
type SubprocessRunner struct {
run subprocessRun
}
// NewSubprocessRunner constructs a production Notarius subprocess runner.
func NewSubprocessRunner() *SubprocessRunner {
return &SubprocessRunner{run: subprocess.Run}
}
// Run executes a complete Notarius pipeline and discovers its published bundle.
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
if r == nil || r.run == nil {
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
}
if err := validateRunRequest(req); err != nil {
return RunResult{}, err
}
args := []string{
"run", req.PipelineID,
"--config", req.ConfigPath,
"--input", req.InputPath,
"--output-dir", req.OutputRoot,
"--json",
}
processResult, err := r.run(ctx, subprocess.RunRequest{
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDirectory,
Timeout: req.Timeout,
DiagnosticOwner: "notarius",
StdoutLogPath: req.ReceiptPath,
StderrLogPath: req.LogPath,
})
baseResult := RunResult{
ReceiptPath: req.ReceiptPath,
LogPath: req.LogPath,
ExitCode: processResult.ExitCode,
Duration: processResult.Duration,
}
if err != nil {
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
}
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
if err != nil {
return baseResult, err
}
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
if err != nil {
return baseResult, err
}
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
if err != nil {
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
}
index, err := loadIndex(bundleRoot, indexPath)
if err != nil {
return baseResult, err
}
rejections, err := loadRejections(index.RejectedPath)
if err != nil {
return baseResult, err
}
warnings, err := loadWarnings(index.WarningsPath)
if err != nil {
return baseResult, err
}
baseResult.Receipt = receipt
baseResult.Index = index
baseResult.BundleRoot = bundleRoot
baseResult.Rejections = rejections
baseResult.Warnings = warnings
return baseResult, nil
}
func validateRunRequest(req RunRequest) error {
if strings.TrimSpace(req.Binary) == "" {
return fmt.Errorf("notarius binary is required")
}
if strings.TrimSpace(req.PipelineID) == "" {
return fmt.Errorf("notarius pipeline id is required")
}
if req.Timeout <= 0 {
return fmt.Errorf("notarius timeout must be positive")
}
for label, path := range map[string]string{
"config": req.ConfigPath,
"input": req.InputPath,
"output root": req.OutputRoot,
"working directory": req.WorkingDirectory,
"receipt": req.ReceiptPath,
"log": req.LogPath,
} {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("notarius %s path is required", label)
}
if !filepath.IsAbs(path) {
return fmt.Errorf("notarius %s path must be absolute", label)
}
}
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
return fmt.Errorf("notarius receipt and log paths must be different")
}
if err := requireRegularFile(req.ConfigPath); err != nil {
return fmt.Errorf("validate notarius config path: %w", err)
}
if err := requireRegularFile(req.InputPath); err != nil {
return fmt.Errorf("validate notarius input path: %w", err)
}
if err := requireDirectory(req.OutputRoot); err != nil {
return fmt.Errorf("validate notarius output root: %w", err)
}
if err := requireDirectory(req.WorkingDirectory); err != nil {
return fmt.Errorf("validate notarius working directory: %w", err)
}
if err := validateLogDestination(req.ReceiptPath); err != nil {
return fmt.Errorf("validate notarius receipt path: %w", err)
}
if err := validateLogDestination(req.LogPath); err != nil {
return fmt.Errorf("validate notarius log path: %w", err)
}
return nil
}
type receiptDocument struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file"`
NormalizedOutputCount *int `json:"normalized_output_count"`
RejectedOutputCount *int `json:"rejected_output_count"`
WarningCount *int `json:"warning_count"`
ValidationStatus string `json:"validation_status"`
DebugDirectory string `json:"debug_directory"`
}
func loadReceipt(path, pipelineID string) (Receipt, error) {
var document receiptDocument
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
}
if document.SchemaVersion != ReceiptSchemaVersion {
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
}
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
document.NormalizedOutputCount == nil ||
document.RejectedOutputCount == nil || document.WarningCount == nil {
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
}
if document.IndexFile != canonicalIndexFile {
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
}
if document.PipelineID != pipelineID {
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
}
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
}
if !filepath.IsAbs(document.OutputDirectory) {
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
}
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
}
return Receipt{
SchemaVersion: document.SchemaVersion,
RunID: document.RunID,
PipelineID: document.PipelineID,
OutputDirectory: filepath.Clean(document.OutputDirectory),
IndexFile: document.IndexFile,
NormalizedOutputCount: *document.NormalizedOutputCount,
RejectedOutputCount: *document.RejectedOutputCount,
WarningCount: *document.WarningCount,
ValidationStatus: document.ValidationStatus,
DebugDirectory: document.DebugDirectory,
}, nil
}
type indexDocument struct {
ManifestFile string `json:"manifest_file"`
OutputFiles *[]laneDocument `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
ChunkMap *pipelineDocument `json:"chunk_map"`
EvidenceContext *pipelineDocument `json:"evidence_context"`
}
type laneDocument struct {
LaneID string `json:"lane_id"`
File string `json:"file"`
MediaType string `json:"media_type"`
ModuleKey string `json:"module_key"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
}
type pipelineDocument struct {
ArtifactKind string `json:"artifact_kind"`
File string `json:"file"`
MediaType string `json:"media_type"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
}
func loadIndex(bundleRoot, indexPath string) (Index, error) {
var document indexDocument
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
return Index{}, fmt.Errorf("decode notarius index: %w", err)
}
for _, field := range []struct {
name string
got string
want string
}{
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
} {
if field.got != field.want {
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
}
}
if document.OutputFiles == nil {
return Index{}, fmt.Errorf("notarius index is missing required output_files")
}
index := Index{
Path: indexPath,
ManifestFile: document.ManifestFile,
RejectedFile: document.RejectedFile,
WarningsFile: document.WarningsFile,
}
var err error
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
}
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
}
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
}
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
for _, lane := range *document.OutputFiles {
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
}
if _, exists := seenLanes[lane.LaneID]; exists {
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
}
seenLanes[lane.LaneID] = struct{}{}
path, err := resolveRegularFile(bundleRoot, lane.File)
if err != nil {
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
}
index.Lanes = append(index.Lanes, LaneDescriptor{
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
SchemaVersion: lane.SchemaVersion,
})
}
if document.ChunkMap != nil {
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
if err != nil {
return Index{}, err
}
}
if document.EvidenceContext != nil {
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
if err != nil {
return Index{}, err
}
}
return index, nil
}
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
}
path, err := resolveRegularFile(bundleRoot, document.File)
if err != nil {
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
}
return &PipelineDescriptor{
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
MediaType: document.MediaType, SchemaID: document.SchemaID,
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
}, nil
}
type rejectionDocument struct {
Rejected *[]struct {
Stage string `json:"stage"`
StepID string `json:"step_id"`
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
ChunkID string `json:"chunk_id"`
ValidatorName string `json:"validator_name"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
} `json:"rejected"`
}
func loadRejections(path string) ([]RejectionSummary, error) {
var document rejectionDocument
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
return nil, fmt.Errorf("decode notarius rejections: %w", err)
}
if document.Rejected == nil {
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
}
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
for _, item := range *document.Rejected {
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
return nil, fmt.Errorf("notarius rejection entries require stage and message")
}
summaries = append(summaries, RejectionSummary{
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
})
}
return summaries, nil
}
type warningDocument struct {
Warnings *[]struct {
Scope string `json:"scope"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
} `json:"warnings"`
}
func loadWarnings(path string) ([]WarningSummary, error) {
var document warningDocument
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
return nil, fmt.Errorf("decode notarius warnings: %w", err)
}
if document.Warnings == nil {
return nil, fmt.Errorf("notarius warning document is missing warnings array")
}
summaries := make([]WarningSummary, 0, len(*document.Warnings))
for _, item := range *document.Warnings {
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
}
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
}
return summaries, nil
}
func decodeBoundedJSON(path string, limit int64, destination any) error {
data, err := fileops.ReadRegularFile(path, limit)
if err != nil {
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
}
if err := json.Unmarshal(data, destination); err != nil {
return err
}
return nil
}
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
root := filepath.Clean(outputRoot)
bundle := filepath.Clean(bundleRoot)
relative, err := filepath.Rel(root, bundle)
if err != nil {
return "", fmt.Errorf("compare notarius output paths: %w", err)
}
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
}
if err := requireDirectoryTree(root, relative); err != nil {
return "", fmt.Errorf("validate notarius output directory: %w", err)
}
return bundle, nil
}
func resolveRegularFile(root, logicalPath string) (string, error) {
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
if err != nil {
return "", err
}
relative, err := filepath.Rel(root, resolved)
if err != nil {
return "", err
}
if err := requireRegularFileTree(root, relative); err != nil {
return "", err
}
return resolved, nil
}
func requireDirectoryTree(root, relative string) error {
if err := requireDirectory(root); err != nil {
return err
}
current := root
for _, component := range strings.Split(relative, string(filepath.Separator)) {
current = filepath.Join(current, component)
if err := requireDirectory(current); err != nil {
return err
}
}
return nil
}
func requireRegularFileTree(root, relative string) error {
components := strings.Split(relative, string(filepath.Separator))
if len(components) == 0 {
return fmt.Errorf("regular file path is required")
}
if err := requireDirectory(root); err != nil {
return err
}
current := root
for _, component := range components[:len(components)-1] {
current = filepath.Join(current, component)
if err := requireDirectory(current); err != nil {
return err
}
}
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
}
func requireDirectory(path string) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("path %q must be a directory without symlinks", path)
}
return nil
}
func requireRegularFile(path string) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf("path %q must be a regular file without symlinks", path)
}
return nil
}
func validateLogDestination(path string) error {
if err := requireDirectory(filepath.Dir(path)); err != nil {
return err
}
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
}
return nil
}

View File

@@ -0,0 +1,572 @@
package notarius
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
)
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
req := validRunRequest(t)
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, true)
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
}}
result, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
wantArgs := []string{
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot, "--json",
}
if !reflect.DeepEqual(captured.Args, wantArgs) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
}
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
t.Fatalf("subprocess request = %#v", captured)
}
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
}
if captured.EnvOverrides != nil {
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
}
for _, arg := range captured.Args {
if arg == "--session-id" {
t.Fatal("subprocess args unexpectedly contain --session-id")
}
}
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
t.Fatalf("receipt = %#v", result.Receipt)
}
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
t.Fatalf("lanes = %#v", result.Index.Lanes)
}
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
}
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
}
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
t.Fatalf("rejections = %#v", result.Rejections)
}
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
t.Fatalf("warnings = %#v", result.Warnings)
}
}
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
req := validRunRequest(t)
writeValidBundleAndReceipt(t, req, false)
receiptFixture := req.ReceiptPath + ".fixture"
data, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
}
if err := os.Remove(req.ReceiptPath); err != nil {
t.Fatalf("Remove(receipt) error = %v", err)
}
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
if err := os.Mkdir(captureDir, 0o755); err != nil {
t.Fatalf("Mkdir(capture) error = %v", err)
}
script := writeShellScript(t, `#!/bin/sh
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
printf 'diagnostic stream\n' >&2
cat "$NOTARIUS_RECEIPT_FIXTURE"
`)
req.Binary = script
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
assertTextFile(t, req.LogPath, "diagnostic stream\n")
receiptBytes, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
if strings.Contains(string(receiptBytes), "diagnostic stream") {
t.Fatal("receipt contains stderr output")
}
}
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
tests := []struct {
name string
scriptBody string
timeout time.Duration
cancel bool
want string
}{
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
req.Timeout = test.timeout
ctx := context.Background()
if test.cancel {
cancelCtx, cancel := context.WithCancel(ctx)
ctx = cancelCtx
time.AfterFunc(20*time.Millisecond, cancel)
}
_, err := NewSubprocessRunner().Run(ctx, req)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
}
if strings.Contains(err.Error(), "decode notarius receipt") {
t.Fatalf("Run() parsed stdout after process failure: %v", err)
}
})
}
}
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
req := validRunRequest(t)
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
t.Fatalf("WriteFile(receipt) error = %v", err)
}
wantErr := errors.New("process failed")
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
}}
_, err := runner.Run(context.Background(), req)
if !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want wrapped process error", err)
}
if strings.Contains(err.Error(), "decode") {
t.Fatalf("Run() parsed receipt after failure: %v", err)
}
}
func TestLoadReceiptValidation(t *testing.T) {
root := t.TempDir()
valid := map[string]any{
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
"validation_status": "approved", "future_field": true,
}
tests := []struct {
name string
mutate func(map[string]any)
raw []byte
wantOK bool
wantError string
}{
{name: "unknown fields tolerated", wantOK: true},
{name: "malformed", raw: []byte("{")},
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
{
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
wantError: `index_file "nested/index.json"`,
},
{
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
wantError: `index_file "./index.json"`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
values := cloneMap(valid)
if test.mutate != nil {
test.mutate(values)
}
if test.raw != nil {
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
} else {
writeJSONFile(t, path, values)
}
_, err := loadReceipt(path, "pipeline-1")
if test.wantOK && err != nil {
t.Fatalf("loadReceipt() error = %v", err)
}
if !test.wantOK && err == nil {
t.Fatal("loadReceipt() error = nil, want validation failure")
}
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
}
})
}
oversized := filepath.Join(root, "oversized.json")
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized) error = %v", err)
}
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadReceipt(oversized) error = %v", err)
}
}
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
root := t.TempDir()
outputRoot := filepath.Join(root, "output")
if err := os.Mkdir(outputRoot, 0o755); err != nil {
t.Fatalf("Mkdir(output root) error = %v", err)
}
validBundle := filepath.Join(outputRoot, "run-1")
if err := os.Mkdir(validBundle, 0o755); err != nil {
t.Fatalf("Mkdir(bundle) error = %v", err)
}
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
t.Fatalf("validateBundleRoot(valid) error = %v", err)
}
outside := filepath.Join(root, "output-other")
if err := os.Mkdir(outside, 0o755); err != nil {
t.Fatalf("Mkdir(outside) error = %v", err)
}
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
t.Run(name, func(t *testing.T) {
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
}
})
}
symlink := filepath.Join(outputRoot, "linked")
if err := os.Symlink(outside, symlink); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
t.Fatal("validateBundleRoot(symlink) error = nil")
}
}
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
tests := []struct {
name string
indexValue any
prepare func(*testing.T, string)
wantError string
}{
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
{name: "renamed manifest", indexValue: func() any {
value := validIndexValue([]any{})
value["manifest_file"] = "metadata.json"
return value
}(), wantError: `manifest_file "metadata.json"`},
{name: "cleanable manifest", indexValue: func() any {
value := validIndexValue([]any{})
value["manifest_file"] = "./manifest.json"
return value
}(), wantError: `manifest_file "./manifest.json"`},
{name: "renamed rejections", indexValue: func() any {
value := validIndexValue([]any{})
value["rejected_file"] = "rejections.json"
return value
}(), wantError: `rejected_file "rejections.json"`},
{name: "renamed warnings", indexValue: func() any {
value := validIndexValue([]any{})
value["warnings_file"] = "diagnostics/warnings.json"
return value
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
{name: "duplicate lane", indexValue: validIndexValue([]any{
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
})},
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
}},
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
}},
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
t.Fatalf("Remove(manifest) error = %v", err)
}
}},
{name: "incomplete pipeline descriptor", indexValue: func() any {
value := validIndexValue([]any{})
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
return value
}()},
{name: "pipeline descriptor escape", indexValue: func() any {
value := validIndexValue([]any{})
value["evidence_context"] = map[string]any{
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
}
return value
}()},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
bundle := createBundleSkeleton(t)
indexPath := filepath.Join(bundle, "index.json")
if raw, ok := test.indexValue.(json.RawMessage); ok {
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
t.Fatalf("WriteFile(index) error = %v", err)
}
} else {
writeJSONFile(t, indexPath, test.indexValue)
}
if test.prepare != nil {
test.prepare(t, bundle)
}
if _, err := loadIndex(bundle, indexPath); err == nil {
t.Fatal("loadIndex() error = nil, want failure")
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
}
})
}
bundle := createBundleSkeleton(t)
oversizedIndex := filepath.Join(bundle, "index.json")
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized index) error = %v", err)
}
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadIndex(oversized) error = %v", err)
}
}
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
root := t.TempDir()
rejectedPath := filepath.Join(root, "rejected.json")
warningsPath := filepath.Join(root, "warnings.json")
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
}}, "future": true})
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
}}, "future": true})
rejections, err := loadRejections(rejectedPath)
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
}
warnings, err := loadWarnings(warningsPath)
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
}
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
t.Run("malformed "+name, func(t *testing.T) {
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
var err error
if name == "rejections" {
_, err = loadRejections(path)
} else {
_, err = loadWarnings(path)
}
if err == nil {
t.Fatal("summary decoder error = nil")
}
})
}
oversized := filepath.Join(root, "oversized.json")
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized) error = %v", err)
}
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadWarnings(oversized) error = %v", err)
}
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadRejections(oversized) error = %v", err)
}
}
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
req := RunRequest{PipelineID: "pipeline"}
want := RunResult{BundleRoot: "/bundle"}
fake := &FakeRunner{Result: want}
got, err := fake.Run(context.Background(), req)
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
}
wantErr := errors.New("configured failure")
fake.Err = wantErr
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
t.Fatalf("Run(configured error) = %v", err)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
before := len(fake.Requests)
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
}
}
func validRunRequest(t *testing.T) RunRequest {
t.Helper()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
inputPath := filepath.Join(root, "input.json")
outputRoot := filepath.Join(root, "outputs")
workingDirectory := filepath.Join(root, "work")
diagnostics := filepath.Join(root, "diagnostics")
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
if err := os.Mkdir(directory, 0o755); err != nil {
t.Fatalf("Mkdir(%q) error = %v", directory, err)
}
}
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
t.Fatalf("WriteFile(config) error = %v", err)
}
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
t.Fatalf("WriteFile(input) error = %v", err)
}
return RunRequest{
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
Timeout: time.Second,
}
}
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
t.Helper()
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
t.Fatalf("MkdirAll(bundle) error = %v", err)
}
for path, data := range map[string]string{
"manifest.json": `{}`,
"lanes/npc.json": `{}`,
"chunk-map.json": `{}`,
"evidence-context.json": `{}`,
} {
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
if includeUnknown {
rejection["future"] = true
warning["future"] = true
}
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
index := validIndexValue([]any{map[string]any{
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
}})
index["chunk_map"] = map[string]any{
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
}
index["evidence_context"] = map[string]any{
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
}
index["future"] = true
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
receipt := map[string]any{
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
}
if includeUnknown {
receipt["future"] = true
}
writeJSONFile(t, req.ReceiptPath, receipt)
}
func createBundleSkeleton(t *testing.T) string {
t.Helper()
bundle := filepath.Join(t.TempDir(), "bundle")
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
t.Fatalf("MkdirAll(bundle) error = %v", err)
}
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", name, err)
}
}
return bundle
}
func validIndexValue(lanes []any) map[string]any {
return map[string]any{
"manifest_file": "manifest.json", "output_files": lanes,
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
}
}
func writeJSONFile(t *testing.T, path string, value any) {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}
func writeShellScript(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "notarius-helper")
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
t.Fatalf("WriteFile(script) error = %v", err)
}
return path
}
func assertTextFile(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
if string(data) != want {
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
}
}
func cloneMap(source map[string]any) map[string]any {
result := make(map[string]any, len(source))
for key, value := range source {
result[key] = value
}
return result
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// NoopRunner is a deterministic no-op scriptorium adapter.
@@ -142,7 +143,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
func materializeRunPlaceholders(req RunArtifactRequest) error {
if req.OutputPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
}
}
@@ -154,17 +155,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
"prompt_id": req.PromptID,
"output_path": req.OutputPath,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
@@ -173,7 +174,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
if req.OutputPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
}
}
@@ -185,17 +186,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
"prompt_id": req.PromptID,
"output_path": req.OutputPath,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}

View File

@@ -9,8 +9,12 @@ import (
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// MaxOutputFileBytes bounds one Scriptorium artifact result.
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
// SubprocessRunner invokes Scriptorium through its public CLI.
type SubprocessRunner struct{}
@@ -52,11 +56,15 @@ func (r *SubprocessRunner) RunArtifact(ctx context.Context, req RunArtifactReque
}
}
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDir,
Timeout: req.Timeout,
EnvOverrides: envOverrides,
SensitiveEnvNames: sensitiveNames,
DiagnosticOwner: "scriptorium",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -134,11 +142,15 @@ func (r *SubprocessRunner) RenderArtifact(ctx context.Context, req RenderArtifac
}
}
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDir,
Timeout: req.Timeout,
EnvOverrides: envOverrides,
SensitiveEnvNames: sensitiveNames,
DiagnosticOwner: "scriptorium",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -223,6 +235,15 @@ func validateCommonRunRequest(
return true, nil
}
func credentialEnvironment(apiKeyEnv string) (map[string]string, []string) {
name := strings.TrimSpace(apiKeyEnv)
if name == "" {
return nil, nil
}
value, _ := os.LookupEnv(name)
return map[string]string{name: value}, []string{name}
}
func buildRunArgs(req RunArtifactRequest) []string {
args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)}
if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" {
@@ -321,18 +342,15 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
"render_format": payload.RenderFormat,
"render_prompt_logged": payload.RenderPromptStore,
}
return subprocess.WriteYAMLAtomic(path, data, 0o644)
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
}
func validateNonEmptyOutput(path string) error {
info, err := os.Stat(path)
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
if err != nil {
return fmt.Errorf("stat file: %w", err)
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
}
if info.IsDir() {
return fmt.Errorf("path is a directory")
}
if info.Size() <= 0 {
if len(data) == 0 {
return fmt.Errorf("file is empty")
}
return nil

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// NoopRunner is a deterministic no-op seriatim adapter.
@@ -68,6 +69,26 @@ func (n *NoopRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
}, nil
}
// Render returns the requested output path with placeholder metadata.
func (n *NoopRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
if err := ctx.Err(); err != nil {
return RenderResult{}, err
}
if err := materializeRenderPlaceholders(req); err != nil {
return RenderResult{}, err
}
return RenderResult{
OutputRenderedPath: req.OutputRenderedPath,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
InvokedBinary: "noop",
Format: req.Format,
Title: req.Title,
Metadata: map[string]any{"placeholder": true},
}, nil
}
// FakeRunner captures merge requests and returns deterministic responses.
type FakeRunner struct {
Requests []MergeRequest
@@ -79,6 +100,9 @@ type FakeRunner struct {
TrimRequests []TrimRequest
TrimErr error
TrimResult TrimResult
RenderRequests []RenderRequest
RenderErr error
RenderResult RenderResult
}
// Run records request and returns configured response.
@@ -195,9 +219,49 @@ func (f *FakeRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
return res, nil
}
// Render records request and returns configured response.
func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
if err := ctx.Err(); err != nil {
return RenderResult{}, err
}
f.RenderRequests = append(f.RenderRequests, req)
if f.RenderErr != nil {
return RenderResult{}, f.RenderErr
}
if err := materializeRenderPlaceholders(req); err != nil {
return RenderResult{}, err
}
res := f.RenderResult
if res.OutputRenderedPath == "" {
res.OutputRenderedPath = req.OutputRenderedPath
}
if res.StdoutLogPath == "" {
res.StdoutLogPath = req.StdoutLogPath
}
if res.StderrLogPath == "" {
res.StderrLogPath = req.StderrLogPath
}
if res.GeneratedConfigPath == "" {
res.GeneratedConfigPath = req.GeneratedConfigPath
}
if res.InvokedBinary == "" {
res.InvokedBinary = "fake"
}
if res.Format == "" {
res.Format = req.Format
}
if res.Title == "" {
res.Title = req.Title
}
if res.Metadata == nil {
res.Metadata = map[string]any{"fake": true}
}
return res, nil
}
func materializePlaceholders(req MergeRequest) error {
if req.OutputMergedTranscriptPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
}
}
@@ -208,22 +272,22 @@ func materializePlaceholders(req MergeRequest) error {
"input_transcript_paths": req.InputTranscriptPaths,
"output_path": req.OutputMergedTranscriptPath,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
if req.ReportPath != "" {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
}
}
@@ -232,7 +296,7 @@ func materializePlaceholders(req MergeRequest) error {
func materializeTrimPlaceholders(req TrimRequest) error {
if req.OutputTrimmedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
}
}
@@ -245,17 +309,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
"output_path": req.OutputTrimmedPath,
"keep_selector": req.KeepSelector,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
@@ -264,7 +328,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
func materializeNormalizePlaceholders(req NormalizeRequest) error {
if req.OutputNormalizedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
}
}
@@ -280,24 +344,60 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
if req.ReportPath != "" {
payload["report_path"] = req.ReportPath
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
if req.ReportPath != "" {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
}
}
return nil
}
func materializeRenderPlaceholders(req RenderRequest) error {
if req.OutputRenderedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
}
}
if req.GeneratedConfigPath != "" {
payload := map[string]any{
"schema": "seriatim.generated.v1",
"placeholder": true,
"command": "render",
"input_path": req.InputTranscriptPath,
"output_path": req.OutputRenderedPath,
"format": req.Format,
"title": req.Title,
"include_timestamps": req.IncludeTimestamps,
"include_segment_ids": req.IncludeSegmentIDs,
"include_metadata": req.IncludeMetadata,
}
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
}
}
if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
}
}
if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
}
}
return nil
}

View File

@@ -148,3 +148,58 @@ func TestFakeRunnerNormalizeError(t *testing.T) {
t.Fatal("expected error, got nil")
}
}
func TestFakeRunnerRenderCapturesRequestAndReturnsPath(t *testing.T) {
fake := &FakeRunner{}
dir := t.TempDir()
req := RenderRequest{
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.render.yml"),
InputTranscriptPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
OutputRenderedPath: filepath.Join(dir, "transcripts", "final.trimmed.md"),
Format: "markdown",
Title: "Session render",
IncludeTimestamps: true,
IncludeSegmentIDs: false,
IncludeMetadata: true,
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.render.stdout.log"),
StderrLogPath: filepath.Join(dir, "logs", "seriatim.render.stderr.log"),
}
res, err := fake.Render(context.Background(), req)
if err != nil {
t.Fatalf("Render() error = %v", err)
}
if len(fake.RenderRequests) != 1 || fake.RenderRequests[0].GeneratedConfigPath == "" {
t.Fatalf("render requests = %#v, want captured request", fake.RenderRequests)
}
if res.OutputRenderedPath != req.OutputRenderedPath {
t.Fatalf("rendered path = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
}
if res.Format != req.Format {
t.Fatalf("format = %q, want %q", res.Format, req.Format)
}
if res.Title != req.Title {
t.Fatalf("title = %q, want %q", res.Title, req.Title)
}
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
if err != nil {
t.Fatalf("read generated config: %v", err)
}
if !strings.Contains(string(cfgData), "command: render") {
t.Fatalf("generated config = %q, want render command marker", string(cfgData))
}
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath, req.OutputRenderedPath} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("expected file %q to exist: %v", path, err)
}
}
}
func TestFakeRunnerRenderError(t *testing.T) {
fake := &FakeRunner{RenderErr: errors.New("boom")}
_, err := fake.Render(context.Background(), RenderRequest{})
if err == nil {
t.Fatal("expected error, got nil")
}
}

View File

@@ -1,4 +1,4 @@
// Package seriatim declares the adapter contract for transcript merge/normalize/trim execution.
// Package seriatim declares the adapter contract for transcript merge/normalize/trim/render execution.
package seriatim
import (
@@ -6,11 +6,12 @@ import (
"time"
)
// Runner is the adapter boundary for seriatim merge/normalize/trim invocations.
// Runner is the adapter boundary for seriatim merge/normalize/trim/render invocations.
type Runner interface {
Run(ctx context.Context, req MergeRequest) (MergeResult, error)
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
Trim(ctx context.Context, req TrimRequest) (TrimResult, error)
Render(ctx context.Context, req RenderRequest) (RenderResult, error)
}
// MergeRequest describes a seriatim merge invocation.
@@ -90,3 +91,33 @@ type TrimResult struct {
KeepSelector string
Metadata map[string]any
}
// RenderRequest describes a seriatim render invocation.
type RenderRequest struct {
Binary string
InputTranscriptPath string
OutputRenderedPath string
Format string
Title string
IncludeTimestamps bool
IncludeSegmentIDs bool
IncludeMetadata bool
StdoutLogPath string
StderrLogPath string
GeneratedConfigPath string
Timeout time.Duration
}
// RenderResult describes a render output.
type RenderResult struct {
OutputRenderedPath string
StdoutLogPath string
StderrLogPath string
GeneratedConfigPath string
ExitCode int
Duration time.Duration
InvokedBinary string
Format string
Title string
Metadata map[string]any
}

View File

@@ -4,14 +4,18 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
// MaxOutputFileBytes bounds each Seriatim JSON or rendered-text result.
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
// EnvConfig defines optional Seriatim environment tuning values.
type EnvConfig struct {
OverlapWordRunGap *float64
@@ -132,6 +136,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req MergeRequest) (MergeResu
Args: args,
Timeout: r.timeout,
EnvOverrides: env,
DiagnosticOwner: "seriatim",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -234,6 +239,7 @@ func (r *SubprocessRunner) Trim(ctx context.Context, req TrimRequest) (TrimResul
Executable: binary,
Args: args,
Timeout: timeout,
DiagnosticOwner: "seriatim",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -322,6 +328,7 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
Executable: binary,
Args: args,
Timeout: timeout,
DiagnosticOwner: "seriatim",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
@@ -384,6 +391,97 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
}, nil
}
// Render executes Seriatim render with deterministic flags and validates non-empty text output.
func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
if r == nil {
return RenderResult{}, fmt.Errorf("seriatim subprocess runner is nil")
}
if strings.TrimSpace(req.InputTranscriptPath) == "" {
return RenderResult{}, fmt.Errorf("seriatim render input path is required")
}
if strings.TrimSpace(req.OutputRenderedPath) == "" {
return RenderResult{}, fmt.Errorf("seriatim render output path is required")
}
format := strings.TrimSpace(req.Format)
if format == "" {
format = "markdown"
}
if format != "markdown" {
return RenderResult{}, fmt.Errorf("seriatim render format %q is unsupported", req.Format)
}
binary := r.binary
if strings.TrimSpace(req.Binary) != "" {
binary = strings.TrimSpace(req.Binary)
}
timeout := r.timeout
if req.Timeout < 0 {
return RenderResult{}, fmt.Errorf("seriatim render timeout must be >= 0")
}
if req.Timeout > 0 {
timeout = req.Timeout
}
args := buildRenderArgs(req, format)
if req.GeneratedConfigPath != "" {
if err := writeRenderInvocationConfig(req, args, binary, timeout, format); err != nil {
return RenderResult{}, fmt.Errorf("write seriatim render invocation config %q: %w", req.GeneratedConfigPath, err)
}
}
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: binary,
Args: args,
Timeout: timeout,
DiagnosticOwner: "seriatim",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
})
if err != nil {
return RenderResult{
OutputRenderedPath: req.OutputRenderedPath,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
ExitCode: runRes.ExitCode,
Duration: runRes.Duration,
InvokedBinary: binary,
Format: format,
Title: req.Title,
}, fmt.Errorf("run seriatim render (binary=%q): %w", binary, err)
}
if err := validateNonEmptyTextFile(req.OutputRenderedPath); err != nil {
return RenderResult{
OutputRenderedPath: req.OutputRenderedPath,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
ExitCode: runRes.ExitCode,
Duration: runRes.Duration,
InvokedBinary: binary,
Format: format,
Title: req.Title,
}, fmt.Errorf("validate seriatim rendered output %q: %w", req.OutputRenderedPath, err)
}
return RenderResult{
OutputRenderedPath: req.OutputRenderedPath,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
ExitCode: runRes.ExitCode,
Duration: runRes.Duration,
InvokedBinary: binary,
Format: format,
Title: req.Title,
Metadata: map[string]any{
"adapter": "seriatim_subprocess",
},
}, nil
}
func (r *SubprocessRunner) buildMergeArgs(req MergeRequest) []string {
args := []string{"merge"}
@@ -455,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
payload["coalesce_gap"] = *r.coalesceGap
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
}
func buildTrimArgs(req TrimRequest) []string {
@@ -480,6 +578,22 @@ func buildNormalizeArgs(req NormalizeRequest, outputSchema string) []string {
return args
}
func buildRenderArgs(req RenderRequest, format string) []string {
args := []string{
"render",
"--input-file", req.InputTranscriptPath,
"--output-file", req.OutputRenderedPath,
"--format", format,
"--include-timestamps=" + strconv.FormatBool(req.IncludeTimestamps),
"--include-segment-ids=" + strconv.FormatBool(req.IncludeSegmentIDs),
"--include-metadata=" + strconv.FormatBool(req.IncludeMetadata),
}
if strings.TrimSpace(req.Title) != "" {
args = append(args, "--title", req.Title)
}
return args
}
func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, timeout time.Duration) error {
payload := map[string]any{
"schema": "seriatim.generated.v1",
@@ -491,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
"output_path": req.OutputTrimmedPath,
"keep_selector": req.KeepSelector,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
}
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
@@ -506,13 +620,31 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
"output_schema": outputSchema,
"report_path": req.ReportPath,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
}
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
payload := map[string]any{
"schema": "seriatim.generated.v1",
"command": "render",
"binary": binary,
"args": args,
"timeout": timeout.String(),
"input_path": req.InputTranscriptPath,
"output_path": req.OutputRenderedPath,
"format": format,
"title": req.Title,
"include_timestamps": req.IncludeTimestamps,
"include_segment_ids": req.IncludeSegmentIDs,
"include_metadata": req.IncludeMetadata,
}
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
}
func validateJSONFile(path string) error {
data, err := os.ReadFile(path)
data, err := readSeriatimResult(path, "JSON output")
if err != nil {
return fmt.Errorf("read file: %w", err)
return err
}
var v any
if err := json.Unmarshal(data, &v); err != nil {
@@ -522,9 +654,9 @@ func validateJSONFile(path string) error {
}
func validateJSONFileWithSegments(path string) error {
data, err := os.ReadFile(path)
data, err := readSeriatimResult(path, "transcript JSON output")
if err != nil {
return fmt.Errorf("read file: %w", err)
return err
}
var payload map[string]any
@@ -541,3 +673,28 @@ func validateJSONFileWithSegments(path string) error {
}
return nil
}
func validateNonEmptyTextFile(path string) error {
data, err := readSeriatimResult(path, "rendered text output")
if err != nil {
return err
}
if len(data) == 0 {
return fmt.Errorf("file is empty")
}
if !utf8.Valid(data) {
return fmt.Errorf("file is not valid utf-8 text")
}
if strings.TrimSpace(string(data)) == "" {
return fmt.Errorf("file has no non-whitespace content")
}
return nil
}
func readSeriatimResult(path, category string) ([]byte, error) {
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
if err != nil {
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
}
return data, nil
}

View File

@@ -569,6 +569,156 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
}
}
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
wrapper := writeHelperWrapper(t)
runner := mustRunner(t, wrapper, false)
req := renderReqForTest(t)
res, err := runner.Render(context.Background(), req)
if err != nil {
t.Fatalf("Render() error = %v", err)
}
if res.OutputRenderedPath != req.OutputRenderedPath {
t.Fatalf("OutputRenderedPath = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
}
if res.Format != req.Format {
t.Fatalf("Format = %q, want %q", res.Format, req.Format)
}
if res.Title != req.Title {
t.Fatalf("Title = %q, want %q", res.Title, req.Title)
}
if res.InvokedBinary != wrapper {
t.Fatalf("InvokedBinary = %q, want %q", res.InvokedBinary, wrapper)
}
if res.ExitCode != 0 {
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
}
if res.Duration <= 0 {
t.Fatalf("Duration = %s, want >0", res.Duration)
}
if res.Metadata == nil || res.Metadata["adapter"] != "seriatim_subprocess" {
t.Fatalf("Metadata = %#v, want adapter marker", res.Metadata)
}
if _, err := os.Stat(req.OutputRenderedPath); err != nil {
t.Fatalf("rendered output missing: %v", err)
}
if _, err := os.Stat(req.StdoutLogPath); err != nil {
t.Fatalf("stdout log missing: %v", err)
}
if _, err := os.Stat(req.StderrLogPath); err != nil {
t.Fatalf("stderr log missing: %v", err)
}
if _, err := os.Stat(req.GeneratedConfigPath); err != nil {
t.Fatalf("generated config missing: %v", err)
}
rec := readHelperRecord(t, recordPath)
wantArgs := []string{
"render",
"--input-file", req.InputTranscriptPath,
"--output-file", req.OutputRenderedPath,
"--format", req.Format,
"--include-timestamps=true",
"--include-segment-ids=true",
"--include-metadata=false",
"--title", req.Title,
}
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
}
}
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
runner := mustRunner(t, writeHelperWrapper(t), false)
req := renderReqForTest(t)
req.Title = ""
if _, err := runner.Render(context.Background(), req); err != nil {
t.Fatalf("Render() error = %v", err)
}
rec := readHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--title" {
t.Fatalf("args = %#v, did not expect --title", rec.Args)
}
}
}
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustRunner(t, writeHelperWrapper(t), false)
req := renderReqForTest(t)
_, err := runner.Render(context.Background(), req)
if err == nil {
t.Fatal("Render() error = nil, want non-nil")
}
if !strings.Contains(err.Error(), "run seriatim render") {
t.Fatalf("error = %q, want subprocess context", err.Error())
}
}
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustRunner(t, writeHelperWrapper(t), false)
req := renderReqForTest(t)
_, err := runner.Render(context.Background(), req)
if err == nil {
t.Fatal("Render() error = nil, want non-nil")
}
if !strings.Contains(err.Error(), "validate seriatim rendered output") {
t.Fatalf("error = %q, want output validation context", err.Error())
}
}
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustRunner(t, writeHelperWrapper(t), false)
req := renderReqForTest(t)
_, err := runner.Render(context.Background(), req)
if err == nil {
t.Fatal("Render() error = nil, want non-nil")
}
if !strings.Contains(err.Error(), "file is empty") {
t.Fatalf("error = %q, want empty-file validation", err.Error())
}
}
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
_, err := NewSubprocessRunnerFromConfigValues("", "10m", "seriatim-intermediate", nil, true, EnvConfig{})
if err == nil {
@@ -702,6 +852,14 @@ func TestSeriatimSubprocessHelper(t *testing.T) {
case "normalize_report_missing":
writeSeriatimHelperFile(outputPath, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
os.Exit(0)
case "render_success":
writeSeriatimHelperFile(outputPath, "# Rendered transcript\n\nHello.\n")
_, _ = os.Stdout.WriteString("seriatim helper render stdout\n")
_, _ = os.Stderr.WriteString("seriatim helper render stderr\n")
os.Exit(0)
case "render_empty_output":
writeSeriatimHelperFile(outputPath, "")
os.Exit(0)
default:
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
os.Exit(2)
@@ -777,6 +935,25 @@ func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
return req
}
func renderReqForTest(t *testing.T) RenderRequest {
t.Helper()
dir := t.TempDir()
input := filepath.Join(dir, "final.trimmed.json")
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
return RenderRequest{
InputTranscriptPath: input,
OutputRenderedPath: filepath.Join(dir, "final.trimmed.md"),
Format: "markdown",
Title: "Session 42",
IncludeTimestamps: true,
IncludeSegmentIDs: true,
IncludeMetadata: false,
GeneratedConfigPath: filepath.Join(dir, "seriatim.render.generated.yml"),
StdoutLogPath: filepath.Join(dir, "seriatim.render.stdout.log"),
StderrLogPath: filepath.Join(dir, "seriatim.render.stderr.log"),
}
}
func mustRunner(t *testing.T, binary string, report bool) *SubprocessRunner {
t.Helper()
coalesce := 3.0

View File

@@ -1,31 +0,0 @@
// Package storage declares archive/storage backend adapter boundaries.
package storage
import "context"
// TODO: implement remote storage/archive backends (S3/SFTP/etc.).
// Backend is the adapter boundary for archive/storage operations.
type Backend interface {
Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error)
}
// ArchiveItem describes one item to archive.
type ArchiveItem struct {
Kind string
LocalPath string
RemoteKey string
}
// ArchiveRequest describes one archive operation.
type ArchiveRequest struct {
SessionID string
ManifestPath string
Items []ArchiveItem
}
// ArchiveResult describes archive operation output.
type ArchiveResult struct {
Archived []ArchiveItem
Metadata map[string]any
}

View File

@@ -0,0 +1,90 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
"math"
"strings"
)
// ReadLimitError reports that a remote object exceeded its caller-owned read
// limit. The limit is enforced against both available object metadata and the
// bytes returned by the opened object body.
type ReadLimitError struct {
Key string
Limit int64
Observed int64
}
func (e *ReadLimitError) Error() string {
return fmt.Sprintf("object %q exceeds %d-byte read limit (observed at least %d bytes)", e.Key, e.Limit, e.Observed)
}
// ReadObjectBounded opens one object version and retains at most maxBytes of
// its content. Object metadata may reject an oversized body early, but a
// limit-plus-one read always enforces the boundary when transfer begins.
func ReadObjectBounded(ctx context.Context, store ObjectStore, key string, maxBytes int64) (info ObjectInfo, data []byte, err error) {
key = strings.TrimSpace(key)
if store == nil {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: store is required")
}
if key == "" {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: key is required")
}
if maxBytes <= 0 || maxBytes == math.MaxInt64 {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: limit must be between 1 and %d bytes", key, int64(math.MaxInt64-1))
}
if err := ctx.Err(); err != nil {
return ObjectInfo{}, nil, err
}
info, body, err := store.Read(ctx, key)
if err != nil {
return ObjectInfo{}, nil, err
}
if body == nil {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: store returned no body", key)
}
defer func() {
if closeErr := body.Close(); closeErr != nil {
data = nil
err = errors.Join(err, fmt.Errorf("close object %q: %w", key, closeErr))
}
}()
if info.Size > maxBytes {
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: info.Size}
}
data, err = io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, maxBytes+1))
if err != nil {
return info, nil, err
}
if err := ctx.Err(); err != nil {
return info, nil, err
}
if int64(len(data)) > maxBytes {
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: int64(len(data))}
}
return info, data, nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
n, err := r.reader.Read(p)
if err == nil {
if contextErr := r.ctx.Err(); contextErr != nil {
return n, contextErr
}
}
return n, err
}

View File

@@ -0,0 +1,135 @@
package storage
import (
"bytes"
"context"
"errors"
"io"
"testing"
)
func TestReadObjectBoundedAcceptsExactLimitWithAbsentSizeMetadata(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 2}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", ETag: "generation"}, body, nil
}}
info, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
if err != nil {
t.Fatalf("ReadObjectBounded() error = %v", err)
}
if string(data) != "12345678" || info.ETag != "generation" {
t.Fatalf("ReadObjectBounded() = (%#v, %q), want opened object metadata and bytes", info, data)
}
if !body.closed {
t.Fatal("object body was not closed")
}
}
func TestReadObjectBoundedRejectsLimitPlusOneDespiteMissingOrInaccurateMetadata(t *testing.T) {
tests := []struct {
name string
metadataSize int64
}{
{name: "missing", metadataSize: 0},
{name: "inaccurate", metadataSize: 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("123456789")), chunkSize: 1}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: test.metadataSize}, body, nil
}}
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
var limitErr *ReadLimitError
if !errors.As(err, &limitErr) {
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
}
if data != nil || body.bytesRead != 9 || !body.closed {
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 9, true", data, body.bytesRead, body.closed)
}
})
}
}
func TestReadObjectBoundedRejectsOversizedMetadataBeforeTransfer(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("small"))}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: 9}, body, nil
}}
_, _, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
var limitErr *ReadLimitError
if !errors.As(err, &limitErr) {
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
}
if body.bytesRead != 0 || !body.closed {
t.Fatalf("bytes read=%d closed=%t, want zero-byte transfer and closed body", body.bytesRead, body.closed)
}
}
func TestReadObjectBoundedPropagatesCancellationAndClosesBody(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 1, afterRead: cancel}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json"}, body, nil
}}
_, data, err := ReadObjectBounded(ctx, store, "control.json", 8)
if !errors.Is(err, context.Canceled) {
t.Fatalf("ReadObjectBounded() error = %v, want context cancellation", err)
}
if data != nil || body.bytesRead != 1 || !body.closed {
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 1, true", data, body.bytesRead, body.closed)
}
}
func TestReadObjectBoundedReturnsCloseFailure(t *testing.T) {
closeErr := errors.New("close failed")
body := &trackingReadCloser{reader: bytes.NewReader([]byte("ok")), closeErr: closeErr}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: 2}, body, nil
}}
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
if !errors.Is(err, closeErr) || data != nil || !body.closed {
t.Fatalf("data=%q error=%v closed=%t, want close failure and no retained data", data, err, body.closed)
}
}
type boundedReadStore struct {
ObjectStore
read func(context.Context, string) (ObjectInfo, io.ReadCloser, error)
}
func (s *boundedReadStore) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
return s.read(ctx, key)
}
type trackingReadCloser struct {
reader io.Reader
chunkSize int
afterRead func()
closeErr error
bytesRead int
closed bool
}
func (r *trackingReadCloser) Read(p []byte) (int, error) {
if r.chunkSize > 0 && len(p) > r.chunkSize {
p = p[:r.chunkSize]
}
n, err := r.reader.Read(p)
r.bytesRead += n
if n > 0 && r.afterRead != nil {
r.afterRead()
r.afterRead = nil
}
return n, err
}
func (r *trackingReadCloser) Close() error {
r.closed = true
return r.closeErr
}

View File

@@ -0,0 +1,23 @@
package storage
import (
"context"
"fmt"
"io"
)
// WriterDownloader is implemented by storage backends that stream an object
// into a caller-owned file handle.
type WriterDownloader interface {
DownloadTo(ctx context.Context, key string, destination io.Writer) error
}
// DownloadTo streams one object into destination. Destination-confined callers
// require this capability rather than granting a backend a mutable pathname.
func DownloadTo(ctx context.Context, store ObjectStore, key string, destination io.Writer) error {
writer, ok := store.(WriterDownloader)
if !ok {
return fmt.Errorf("object store does not support handle-confined downloads")
}
return writer.DownloadTo(ctx, key, destination)
}

View File

@@ -14,16 +14,15 @@ func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectSt
return nil, fmt.Errorf("pipeline config is required")
}
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
switch strings.ToLower(strings.TrimSpace(cfg.Pipeline.Storage.Backend)) {
case config.StorageBackendS3:
if cfg.Pipeline.Storage.S3 == nil {
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
}
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
case "", config.StorageBackendLocal:
return nil, fmt.Errorf("no remote object store backend is configured")
default:
return nil, fmt.Errorf("unsupported pipeline.storage.backend %q", cfg.Pipeline.Storage.Backend)
}
}

View File

@@ -54,3 +54,35 @@ func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
}
}
func TestNewObjectStoreFromConfigDoesNotInferS3FromProviderFields(t *testing.T) {
called := false
original := newS3Client
t.Cleanup(func() { newS3Client = original })
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
called = true
return &fakeS3API{}, nil
}
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{
Backend: config.StorageBackendLocal,
S3: &config.StorageS3Config{Bucket: "my-archive"},
}},
})
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
}
if called {
t.Fatal("NewObjectStoreFromConfig() constructed S3 from incidental provider fields")
}
}
func TestNewObjectStoreFromConfigRejectsUnknownBackend(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{Backend: "s33"}},
})
if err == nil || !strings.Contains(err.Error(), "unsupported pipeline.storage.backend") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want unsupported-backend error", err)
}
}

View File

@@ -1,40 +1,35 @@
package storage
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
// NoopBackend is a deterministic no-op archive/storage adapter.
type NoopBackend struct{}
// Archive returns the requested items as archived with placeholder metadata.
func (n *NoopBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
if err := ctx.Err(); err != nil {
return ArchiveResult{}, err
}
return ArchiveResult{Archived: append([]ArchiveItem(nil), req.Items...), Metadata: map[string]any{"placeholder": true}}, nil
}
// FakeBackend captures archive requests and returns deterministic responses.
// FakeBackend provides a deterministic in-memory object store for tests.
type FakeBackend struct {
Requests []ArchiveRequest
Err error
Result ArchiveResult
mu sync.RWMutex
Objects map[string]FakeObject
Uploads []FakeUploadCall
Downloads []FakeDownloadCall
Reads []FakeReadCall
ListErr error
DownloadErr error
UploadErr error
ExistsErr error
UploadHook func(FakeUploadCall) error
DownloadHook func(FakeDownloadCall) error
}
// FakeUploadCall captures one upload invocation in call order.
@@ -48,25 +43,12 @@ type FakeUploadCall struct {
type FakeDownloadCall struct {
Key string
LocalPath string
Bytes int64
}
// Archive records request and returns configured response.
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
if err := ctx.Err(); err != nil {
return ArchiveResult{}, err
}
f.Requests = append(f.Requests, req)
if f.Err != nil {
return ArchiveResult{}, f.Err
}
res := f.Result
if res.Archived == nil {
res.Archived = append([]ArchiveItem(nil), req.Items...)
}
if res.Metadata == nil {
res.Metadata = map[string]any{"fake": true}
}
return res, nil
// FakeReadCall captures one opened object in call order.
type FakeReadCall struct {
Key string
}
// FakeObject is a deterministic fake object-store record.
@@ -80,6 +62,12 @@ type FakeObject struct {
// SeedObject inserts or replaces an object in the fake object store.
func (f *FakeBackend) SeedObject(obj FakeObject) {
f.mu.Lock()
defer f.mu.Unlock()
f.seedObject(obj)
}
func (f *FakeBackend) seedObject(obj FakeObject) {
if f.Objects == nil {
f.Objects = map[string]FakeObject{}
}
@@ -87,6 +75,9 @@ func (f *FakeBackend) SeedObject(obj FakeObject) {
obj.Key = key
obj.Data = append([]byte(nil), obj.Data...)
obj.Metadata = copyMetadata(obj.Metadata)
if obj.ETag == "" {
obj.ETag = fakeObjectETag(obj.Data)
}
f.Objects[key] = obj
}
@@ -99,6 +90,8 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
return nil, f.ListErr
}
f.mu.RLock()
defer f.mu.RUnlock()
normalizedPrefix := normalizeObjectKey(prefix)
keys := make([]string, 0, len(f.Objects))
for key := range f.Objects {
@@ -121,34 +114,77 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
return out, nil
}
// Download writes one object to a local path.
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
// Read returns a stable object body and the generation observed with it.
func (f *FakeBackend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, nil, err
}
if f.DownloadErr != nil {
return ObjectInfo{}, nil, f.DownloadErr
}
normalizedKey := normalizeObjectKey(key)
f.mu.RLock()
obj, ok := f.Objects[normalizedKey]
if ok {
obj.Data = append([]byte(nil), obj.Data...)
obj.Metadata = copyMetadata(obj.Metadata)
}
f.mu.RUnlock()
if !ok {
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
}
f.mu.Lock()
f.Reads = append(f.Reads, FakeReadCall{Key: normalizedKey})
f.mu.Unlock()
return ObjectInfo{Key: obj.Key, Size: int64(len(obj.Data)), ETag: obj.ETag, LastModified: obj.LastModified}, io.NopCloser(bytes.NewReader(obj.Data)), nil
}
// DownloadTo writes one object to a caller-owned destination writer.
func (f *FakeBackend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
if err := ctx.Err(); err != nil {
return err
}
if f.DownloadErr != nil {
return f.DownloadErr
}
if destination == nil {
return fmt.Errorf("download object: destination writer is required")
}
if f.DownloadHook != nil {
if err := f.DownloadHook(FakeDownloadCall{Key: normalizeObjectKey(key)}); err != nil {
return err
}
}
_, source, err := f.Read(ctx, key)
if err != nil {
return err
}
defer source.Close()
count, err := io.Copy(destination, source)
if err != nil {
return fmt.Errorf("download object %q: write destination: %w", key, err)
}
f.mu.Lock()
f.Downloads = append(f.Downloads, FakeDownloadCall{Key: normalizeObjectKey(key), Bytes: count})
f.mu.Unlock()
return nil
}
// Download writes one object to a local path.
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
obj, ok := f.Objects[normalizeObjectKey(key)]
if !ok {
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
}
f.Downloads = append(f.Downloads, FakeDownloadCall{
Key: normalizeObjectKey(key),
LocalPath: localPath,
})
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
}
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
return fmt.Errorf("download object %q: write local file: %w", key, err)
destination, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("download object %q: create local file: %w", key, err)
}
return nil
defer destination.Close()
return f.DownloadTo(ctx, key, destination)
}
// Upload reads a local file and stores it under key.
@@ -166,33 +202,117 @@ func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts Up
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := os.ReadFile(localPath)
file, err := os.Open(localPath)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
}
defer file.Close()
return f.uploadReader(ctx, file, key, opts, localPath)
}
// UploadReader stores content provided by a caller-owned reader.
func (f *FakeBackend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
return f.uploadReader(ctx, source, key, opts, "reader")
}
func (f *FakeBackend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, localPath string) (ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, err
}
if f.UploadErr != nil {
return ObjectInfo{}, f.UploadErr
}
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
}
if strings.TrimSpace(key) == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := io.ReadAll(source)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
}
normalizedKey := normalizeObjectKey(key)
f.Uploads = append(f.Uploads, FakeUploadCall{
call := FakeUploadCall{
LocalPath: localPath,
Key: normalizedKey,
Options: UploadOptions{
Metadata: copyMetadata(opts.Metadata),
ContentType: opts.ContentType,
},
})
now := time.Now().UTC()
obj := FakeObject{
Key: normalizedKey,
Data: data,
Metadata: copyMetadata(opts.Metadata),
LastModified: &now,
}
f.SeedObject(obj)
return ObjectInfo{
Key: normalizedKey,
Size: int64(len(data)),
LastModified: &now,
}, nil
f.mu.Lock()
f.Uploads = append(f.Uploads, call)
f.mu.Unlock()
if f.UploadHook != nil {
if err := f.UploadHook(call); err != nil {
return ObjectInfo{}, err
}
}
return f.storeUploadedObject(normalizedKey, data, opts), nil
}
// UploadConditional atomically checks and replaces one mutable object.
func (f *FakeBackend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
if err := ctx.Err(); err != nil {
return ObjectInfo{}, err
}
if err := validateWriteCondition(condition); err != nil {
return ObjectInfo{}, err
}
if f.UploadErr != nil {
return ObjectInfo{}, f.UploadErr
}
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
}
normalizedKey := normalizeObjectKey(key)
if normalizedKey == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
data, err := io.ReadAll(source)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
}
call := FakeUploadCall{Key: normalizedKey, Options: UploadOptions{Metadata: copyMetadata(opts.Metadata), ContentType: opts.ContentType}}
f.mu.Lock()
f.Uploads = append(f.Uploads, call)
f.mu.Unlock()
if f.UploadHook != nil {
if err := f.UploadHook(call); err != nil {
return ObjectInfo{}, err
}
}
f.mu.Lock()
defer f.mu.Unlock()
existing, found := f.Objects[normalizedKey]
if condition.RequireAbsent && found {
return ObjectInfo{}, ErrConditionNotMet
}
if expected := strings.TrimSpace(condition.MatchETag); expected != "" && (!found || existing.ETag != expected) {
return ObjectInfo{}, ErrConditionNotMet
}
return f.storeUploadedObjectLocked(normalizedKey, data, opts), nil
}
func (f *FakeBackend) storeUploadedObject(key string, data []byte, opts UploadOptions) ObjectInfo {
f.mu.Lock()
defer f.mu.Unlock()
return f.storeUploadedObjectLocked(key, data, opts)
}
func (f *FakeBackend) storeUploadedObjectLocked(key string, data []byte, opts UploadOptions) ObjectInfo {
now := time.Now().UTC()
obj := FakeObject{Key: key, Data: append([]byte(nil), data...), Metadata: copyMetadata(opts.Metadata), LastModified: &now}
f.seedObject(obj)
return ObjectInfo{Key: key, Size: int64(len(data)), ETag: fakeObjectETag(data), LastModified: &now}
}
func fakeObjectETag(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// Exists checks object presence.
@@ -203,7 +323,9 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
if f.ExistsErr != nil {
return false, f.ExistsErr
}
f.mu.RLock()
_, ok := f.Objects[normalizeObjectKey(key)]
f.mu.RUnlock()
return ok, nil
}

View File

@@ -9,30 +9,6 @@ import (
"testing"
)
func TestFakeBackendCapturesRequestAndReturnsItems(t *testing.T) {
fake := &FakeBackend{}
req := ArchiveRequest{SessionID: "s1", Items: []ArchiveItem{{Kind: "artifact", LocalPath: "artifacts/log.md"}}}
res, err := fake.Archive(context.Background(), req)
if err != nil {
t.Fatalf("Archive() error = %v", err)
}
if len(fake.Requests) != 1 || fake.Requests[0].SessionID != "s1" {
t.Fatalf("requests = %#v, want captured request", fake.Requests)
}
if len(res.Archived) != 1 {
t.Fatalf("archived len = %d, want 1", len(res.Archived))
}
}
func TestFakeBackendError(t *testing.T) {
fake := &FakeBackend{Err: errors.New("boom")}
_, err := fake.Archive(context.Background(), ArchiveRequest{})
if err == nil {
t.Fatal("expected error, got nil")
}
}
func TestFakeBackendListPrefixFiltering(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
@@ -95,6 +71,21 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
}
}
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
fake := &FakeBackend{}
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
old := fake.Objects["locks.yml"].ETag
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
t.Fatalf("UploadConditional() error = %v", err)
}
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
}
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
t.Fatalf("locks object = %q, want successful replacement preserved", got)
}
}
func TestFakeBackendObjectErrors(t *testing.T) {
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}

View File

@@ -2,18 +2,32 @@ package storage
import (
"context"
"errors"
"io"
"time"
)
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
// ErrConditionNotMet reports that an object changed or already existed before a
// conditional write could be committed.
var ErrConditionNotMet = errors.New("object write condition not met")
// ReaderUploader streams caller-owned, already-opened content to object storage.
// Callers retain source-selection and filesystem-confinement policy.
type ReaderUploader interface {
UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error)
}
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
//
// Key invariant:
// callers pass full bucket-relative object keys. Backend implementations do not
// infer Narratio session semantics and do not prepend root prefixes.
type ObjectStore interface {
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error)
Download(ctx context.Context, key, localPath string) error
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
Exists(ctx context.Context, key string) (bool, error)
}
@@ -30,3 +44,10 @@ type UploadOptions struct {
Metadata map[string]string
ContentType string
}
// WriteCondition protects a mutable object update against a stale snapshot.
// Exactly one condition is required by UploadConditional.
type WriteCondition struct {
MatchETag string
RequireAbsent bool
}

View File

@@ -117,6 +117,7 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
normalizedPrefix := normalizeObjectKey(prefix)
out := make([]ObjectInfo, 0)
var token *string
seenTokens := map[string]struct{}{}
for {
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
@@ -142,44 +143,78 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
})
}
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
if !valueOrFalseBool(resp.IsTruncated) {
break
}
token = resp.NextContinuationToken
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
if next == "" {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
}
if _, repeated := seenTokens[next]; repeated {
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
}
seenTokens[next] = struct{}{}
token = &next
}
return out, nil
}
// Read retrieves an object together with the generation observed for its body.
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
normalizedKey := normalizeObjectKey(key)
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
if err != nil {
if isS3NotFound(err) {
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
}
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
}
var lastModified *time.Time
if resp.LastModified != nil {
t := *resp.LastModified
lastModified = &t
}
return ObjectInfo{
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
}, resp.Body, nil
}
// DownloadTo retrieves one object into the caller-owned destination writer.
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
if destination == nil {
return fmt.Errorf("download object: destination writer is required")
}
_, body, err := b.Read(ctx, key)
if err != nil {
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
}
defer body.Close()
if _, err := io.Copy(destination, body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
}
return nil
}
// Download retrieves one object to localPath, creating parent directories as needed.
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
normalizedKey := normalizeObjectKey(key)
if strings.TrimSpace(localPath) == "" {
return fmt.Errorf("download object: local path is required")
}
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
})
if err != nil {
return fmt.Errorf("download object %q: %w", normalizedKey, err)
}
defer resp.Body.Close()
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
}
dst, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
return fmt.Errorf("download object %q: create local file: %w", key, err)
}
defer dst.Close()
if _, err := io.Copy(dst, resp.Body); err != nil {
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
if err := b.DownloadTo(ctx, key, dst); err != nil {
return err
}
if err := dst.Sync(); err != nil {
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
return fmt.Errorf("download object %q: sync local file: %w", key, err)
}
return nil
}
@@ -204,28 +239,65 @@ func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts Uplo
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
}
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
}
// UploadReader sends caller-owned content to key.
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
}
// UploadConditional uploads a mutable object only when its observed generation
// still matches, or when no object exists yet.
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
if err := validateWriteCondition(condition); err != nil {
return ObjectInfo{}, err
}
return b.uploadReader(ctx, source, key, opts, 0, condition)
}
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
normalizedKey := normalizeObjectKey(key)
if source == nil {
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
}
if normalizedKey == "" {
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
}
input := &s3.PutObjectInput{
Bucket: &b.bucket,
Key: &normalizedKey,
Body: file,
Body: source,
Metadata: copyMetadata(opts.Metadata),
}
if strings.TrimSpace(opts.ContentType) != "" {
ct := strings.TrimSpace(opts.ContentType)
input.ContentType = &ct
}
if condition.RequireAbsent {
wildcard := "*"
input.IfNoneMatch = &wildcard
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
input.IfMatch = &expected
}
resp, err := b.client.PutObject(ctx, input)
if err != nil {
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
if isS3ConditionalConflict(err) {
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
}
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
}
return ObjectInfo{
info := ObjectInfo{
Key: normalizedKey,
Size: stat.Size(),
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
}, nil
}
if size > 0 {
info.Size = size
}
return info, nil
}
// Exists checks whether one object key exists.
@@ -239,18 +311,43 @@ func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
return true, nil
}
if isS3NotFound(err) {
return false, nil
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
}
func validateWriteCondition(condition WriteCondition) error {
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
}
return nil
}
func isS3NotFound(err error) bool {
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
return true
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NotFound", "NoSuchKey", "404":
return false, nil
return true
}
}
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
return false
}
func isS3ConditionalConflict(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
return true
}
}
return false
}
func valueOrEmpty(v *string) string {

View File

@@ -2,6 +2,7 @@ package storage
import (
"context"
"errors"
"io"
"os"
"path/filepath"
@@ -18,10 +19,15 @@ import (
type fakeS3API struct {
listOut *s3.ListObjectsV2Output
listOutputs []*s3.ListObjectsV2Output
listErr error
listCalls int
getBody io.ReadCloser
getErr error
getSize *int64
getETag *string
getLastModified *time.Time
putOut *s3.PutObjectOutput
putErr error
@@ -29,6 +35,7 @@ type fakeS3API struct {
headErr error
lastList *s3.ListObjectsV2Input
lastLists []*s3.ListObjectsV2Input
lastGet *s3.GetObjectInput
lastPut *s3.PutObjectInput
lastHead *s3.HeadObjectInput
@@ -36,15 +43,56 @@ type fakeS3API struct {
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
f.lastList = params
f.lastLists = append(f.lastLists, params)
if f.listErr != nil {
return nil, f.listErr
}
if f.listCalls < len(f.listOutputs) {
out := f.listOutputs[f.listCalls]
f.listCalls++
return out, nil
}
if f.listOut == nil {
return &s3.ListObjectsV2Output{}, nil
}
return f.listOut, nil
}
func TestS3BackendListPaginatesAndRejectsNonProgressingTokens(t *testing.T) {
t.Run("multiple pages", func(t *testing.T) {
client := &fakeS3API{listOutputs: []*s3.ListObjectsV2Output{
{Contents: []types.Object{{Key: strPtr("prefix/a"), Size: int64Ptr(1)}}, IsTruncated: boolPtr(true), NextContinuationToken: strPtr("next")},
{Contents: []types.Object{{Key: strPtr("prefix/b"), Size: int64Ptr(2)}}, IsTruncated: boolPtr(false)},
}}
items, err := (&S3Backend{bucket: "bucket-1", client: client}).List(context.Background(), "prefix/")
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(items) != 2 || items[0].Key != "prefix/a" || items[1].Key != "prefix/b" {
t.Fatalf("List() items = %#v", items)
}
if len(client.lastLists) != 2 || client.lastLists[1].ContinuationToken == nil || *client.lastLists[1].ContinuationToken != "next" {
t.Fatalf("continuation calls = %#v", client.lastLists)
}
})
for _, test := range []struct {
name string
outputs []*s3.ListObjectsV2Output
want string
}{
{name: "empty", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true)}}, want: "empty continuation token"},
{name: "repeated", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}, {IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}}, want: "repeated continuation token"},
} {
t.Run(test.name, func(t *testing.T) {
_, err := (&S3Backend{bucket: "bucket-1", client: &fakeS3API{listOutputs: test.outputs}}).List(context.Background(), "prefix/")
if err == nil || !strings.Contains(err.Error(), test.want) || !strings.Contains(err.Error(), "bucket-1") || !strings.Contains(err.Error(), "prefix/") {
t.Fatalf("List() error = %v, want contextual %q", err, test.want)
}
})
}
}
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
f.lastGet = params
if f.getErr != nil {
@@ -54,7 +102,7 @@ func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ..
if body == nil {
body = io.NopCloser(strings.NewReader(""))
}
return &s3.GetObjectOutput{Body: body}, nil
return &s3.GetObjectOutput{Body: body, ContentLength: f.getSize, ETag: f.getETag, LastModified: f.getLastModified}, nil
}
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
@@ -126,6 +174,33 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
}
}
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
client := &fakeS3API{
getBody: io.NopCloser(strings.NewReader("locks")),
getSize: int64Ptr(5),
getETag: strPtr(`"generation"`),
getLastModified: &lastModified,
}
backend := &S3Backend{bucket: "bucket-1", client: client}
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
if err != nil {
t.Fatalf("Read() error = %v", err)
}
data, readErr := io.ReadAll(body)
closeErr := body.Close()
if readErr != nil || closeErr != nil {
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
}
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
t.Fatalf("Read() info = %#v, want opened object metadata", info)
}
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
}
}
func TestS3BackendUploadAndExists(t *testing.T) {
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
backend := &S3Backend{bucket: "bucket-1", client: client}
@@ -160,6 +235,22 @@ func TestS3BackendUploadAndExists(t *testing.T) {
}
}
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
backend := &S3Backend{bucket: "bucket-1", client: client}
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
t.Fatalf("UploadConditional() error = %v", err)
}
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
}
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
if !errors.Is(err, ErrConditionNotMet) {
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
}
}
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
@@ -249,5 +340,6 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
func strPtr(v string) *string { return &v }
func int64Ptr(v int64) *int64 { return &v }
func boolPtr(v bool) *bool { return &v }
var _ s3API = (*fakeS3API)(nil)

View File

@@ -0,0 +1,36 @@
package storage
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
)
// DownloadObjectToTemp downloads an object into a temporary file and returns
// the cleaned local path.
func DownloadObjectToTemp(ctx context.Context, store ObjectStore, key, pattern string) (string, error) {
if store == nil {
return "", fmt.Errorf("object store is required")
}
if strings.TrimSpace(pattern) == "" {
return "", fmt.Errorf("temp file pattern is required")
}
tmp, err := os.CreateTemp("", pattern)
if err != nil {
return "", fmt.Errorf("create temp file: %w", err)
}
path := tmp.Name()
if err := tmp.Close(); err != nil {
_ = os.Remove(path)
return "", fmt.Errorf("close temp file: %w", err)
}
if err := store.Download(ctx, key, path); err != nil {
_ = os.Remove(path)
return "", err
}
return filepath.Clean(path), nil
}

View File

@@ -0,0 +1,69 @@
package storage
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
func TestDownloadObjectToTempSuccess(t *testing.T) {
store := &FakeBackend{}
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
if err != nil {
t.Fatalf("DownloadObjectToTemp() error = %v", err)
}
t.Cleanup(func() { _ = os.Remove(path) })
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "run-123\n" {
t.Fatalf("downloaded data = %q, want %q", string(data), "run-123\n")
}
}
func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
sentinel := errors.New("download failed")
store := &FakeBackend{DownloadErr: sentinel}
pattern := "narratio-test-fail-*.txt"
before, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
if err != nil {
t.Fatalf("Glob(before) error = %v", err)
}
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
if !errors.Is(err, sentinel) {
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
}
if strings.TrimSpace(path) != "" {
t.Fatalf("DownloadObjectToTemp() path = %q, want empty on failure", path)
}
after, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
if err != nil {
t.Fatalf("Glob(after) error = %v", err)
}
if len(after) != len(before) {
t.Fatalf("temp file count changed after failed download: before=%d after=%d", len(before), len(after))
}
}
func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
sentinel := errors.New("object missing")
store := &FakeBackend{DownloadErr: sentinel}
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
if err == nil {
t.Fatal("DownloadObjectToTemp() error = nil, want error")
}
err = fmt.Errorf("download run pointer failed: %w", err)
if !errors.Is(err, sentinel) {
t.Fatalf("wrapped error does not preserve sentinel cause: %v", err)
}
}

View File

@@ -0,0 +1,391 @@
package subprocess
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
const (
// MaxStdoutDiagnosticBytes bounds persisted stdout from one external command.
MaxStdoutDiagnosticBytes int64 = 8 * 1024 * 1024
// MaxStderrDiagnosticBytes bounds persisted stderr from one external command.
MaxStderrDiagnosticBytes int64 = 8 * 1024 * 1024
diagnosticTailBytes = 2048
)
var inheritedEnvironmentNames = map[string]struct{}{
"COMSPEC": {},
"HOME": {},
"PATH": {},
"SYSTEMROOT": {},
"TMP": {},
"TMPDIR": {},
"TEMP": {},
"WINDIR": {},
// These test-only helper destinations let the adapter package tests exercise
// real command invocation without widening the production environment.
"AUDITA_HELPER_RECORD_PATH": {},
"AUDITA_HELPER_MODE": {},
"GO_WANT_AUDITA_HELPER": {},
"GO_WANT_SCRIPTORIUM_HELPER": {},
"GO_WANT_SERIATIM_HELPER": {},
"GO_WANT_SUBPROCESS_HELPER": {},
"NOTARIUS_CAPTURE_DIR": {},
"NOTARIUS_RECEIPT_FIXTURE": {},
"SCRIPTORIUM_HELPER_RECORD_PATH": {},
"SCRIPTORIUM_HELPER_MODE": {},
"SERIATIM_HELPER_RECORD_PATH": {},
"SERIATIM_HELPER_MODE": {},
}
var sensitiveEnvironmentNames = map[string]struct{}{
"ANTHROPIC_API_KEY": {},
"API_KEY": {},
"AUDITA_LLM_API_KEY": {},
"AWS_ACCESS_KEY_ID": {},
"AWS_SECRET_ACCESS_KEY": {},
"AWS_SESSION_TOKEN": {},
"OPENAI_API_KEY": {},
"OPENROUTER_API_KEY": {},
}
type captureLimitError struct {
stream string
owner string
limit int64
}
func (e *captureLimitError) Error() string {
return fmt.Sprintf("%s diagnostic capture for %s exceeded %d bytes", e.stream, e.owner, e.limit)
}
type logWriters struct {
files []*os.File
Stdout io.Writer
Stderr io.Writer
limits chan *captureLimitError
mu sync.Mutex
limit *captureLimitError
stdout *diagnosticWriter
stderr *diagnosticWriter
}
type diagnosticWriter struct {
logs *logWriters
stream string
owner string
target io.Writer
limit int64
received int64
persisted int64
redactor streamRedactor
tail []byte
}
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
cleanStdout := cleanLogPath(stdoutPath)
cleanStderr := cleanLogPath(stderrPath)
stdoutFile, err := openDiagnosticFile(cleanStdout)
if err != nil {
return nil, fmt.Errorf("open stdout log: %w", err)
}
stderrFile := stdoutFile
if cleanStdout != cleanStderr {
stderrFile, err = openDiagnosticFile(cleanStderr)
if err != nil {
_ = stdoutFile.Close()
return nil, fmt.Errorf("open stderr log: %w", err)
}
}
if cleanStdout == cleanStderr {
logs.files = []*os.File{stdoutFile}
} else {
logs.files = []*os.File{stdoutFile, stderrFile}
}
logs.stdout = newDiagnosticWriter(logs, "stdout", owner, stdoutFile, MaxStdoutDiagnosticBytes, sensitiveValues)
logs.stderr = newDiagnosticWriter(logs, "stderr", owner, stderrFile, MaxStderrDiagnosticBytes, sensitiveValues)
logs.Stdout = logs.stdout
logs.Stderr = logs.stderr
return logs, nil
}
func newDiagnosticWriter(logs *logWriters, stream, owner string, target io.Writer, limit int64, sensitiveValues []string) *diagnosticWriter {
return &diagnosticWriter{
logs: logs,
stream: stream,
owner: owner,
target: target,
limit: limit,
redactor: newStreamRedactor(sensitiveValues),
}
}
func (w *diagnosticWriter) Write(data []byte) (int, error) {
if w.received >= w.limit {
return len(data), w.reachLimit()
}
accepted := data
if remaining := w.limit - w.received; int64(len(accepted)) > remaining {
accepted = accepted[:remaining]
}
w.received += int64(len(accepted))
if err := w.writeRedacted(w.redactor.Write(accepted)); err != nil {
return len(data), err
}
if len(accepted) != len(data) {
return len(data), w.reachLimit()
}
return len(data), nil
}
func (w *diagnosticWriter) Flush() error {
return w.writeRedacted(w.redactor.Flush())
}
func (w *diagnosticWriter) writeRedacted(data []byte) error {
if len(data) == 0 {
return nil
}
w.logs.mu.Lock()
remaining := w.limit - w.persisted
if remaining <= 0 {
w.logs.mu.Unlock()
return w.reachLimit()
}
toWrite := data
exceeded := int64(len(data)) > remaining
if exceeded {
toWrite = toWrite[:remaining]
}
written, err := w.target.Write(toWrite)
w.persisted += int64(written)
w.retainTail(toWrite[:written])
w.logs.mu.Unlock()
if err != nil {
return err
}
if exceeded {
return w.reachLimit()
}
return nil
}
func (w *diagnosticWriter) Tail() string {
w.logs.mu.Lock()
defer w.logs.mu.Unlock()
return strings.TrimSpace(string(w.tail))
}
func (w *diagnosticWriter) retainTail(data []byte) {
if len(data) >= diagnosticTailBytes {
if cap(w.tail) < diagnosticTailBytes {
w.tail = make([]byte, diagnosticTailBytes)
} else {
w.tail = w.tail[:diagnosticTailBytes]
}
copy(w.tail, data[len(data)-diagnosticTailBytes:])
return
}
if cap(w.tail) < diagnosticTailBytes {
retained := make([]byte, len(w.tail), diagnosticTailBytes)
copy(retained, w.tail)
w.tail = retained
}
if overflow := len(w.tail) + len(data) - diagnosticTailBytes; overflow > 0 {
copy(w.tail, w.tail[overflow:])
w.tail = w.tail[:len(w.tail)-overflow]
}
w.tail = append(w.tail, data...)
}
func (w *diagnosticWriter) reachLimit() error {
limit := &captureLimitError{stream: w.stream, owner: w.owner, limit: w.limit}
w.logs.mu.Lock()
if w.logs.limit == nil {
w.logs.limit = limit
w.logs.limits <- limit
}
w.logs.mu.Unlock()
return limit
}
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
func (l *logWriters) Limit() *captureLimitError {
l.mu.Lock()
defer l.mu.Unlock()
return l.limit
}
func (l *logWriters) Flush() error {
return joinErrors(l.stdout.Flush(), l.stderr.Flush())
}
func (l *logWriters) Close() {
_ = l.Flush()
for _, file := range l.files {
_ = file.Close()
}
}
func cleanLogPath(path string) string {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return ""
}
return filepath.Clean(trimmed)
}
func openDiagnosticFile(path string) (*os.File, error) {
if path == "" {
return os.OpenFile(os.DevNull, os.O_WRONLY, 0)
}
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
}
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
if err != nil {
return nil, fmt.Errorf("open log file %q: %w", path, err)
}
if err := file.Chmod(fileops.WorkspaceFileMode); err != nil {
_ = file.Close()
return nil, fmt.Errorf("set log file permissions %q: %w", path, err)
}
return file, nil
}
func (r RunRequest) diagnosticOwner() string {
if owner := strings.TrimSpace(r.DiagnosticOwner); owner != "" {
return owner
}
return "subprocess"
}
func buildChildEnvironment(base []string, overrides map[string]string) []string {
values := make(map[string]string, len(inheritedEnvironmentNames)+len(overrides))
for _, item := range base {
name, value, ok := strings.Cut(item, "=")
if !ok {
continue
}
normalized := strings.ToUpper(name)
if _, allowed := inheritedEnvironmentNames[normalized]; allowed {
values[name] = value
}
}
for name, value := range overrides {
values[name] = value
}
names := make([]string, 0, len(values))
for name := range values {
names = append(names, name)
}
sort.Strings(names)
out := make([]string, 0, len(names))
for _, name := range names {
out = append(out, name+"="+values[name])
}
return out
}
func sensitiveEnvironmentValues(environment []string, additionalNames []string) []string {
names := make(map[string]struct{}, len(sensitiveEnvironmentNames)+len(additionalNames))
for name := range sensitiveEnvironmentNames {
names[name] = struct{}{}
}
for _, name := range additionalNames {
if trimmed := strings.ToUpper(strings.TrimSpace(name)); trimmed != "" {
names[trimmed] = struct{}{}
}
}
values := make([]string, 0, len(names))
for _, item := range environment {
name, value, ok := strings.Cut(item, "=")
if !ok || strings.TrimSpace(value) == "" {
continue
}
if _, sensitive := names[strings.ToUpper(name)]; sensitive {
values = append(values, value)
}
}
return values
}
type streamRedactor struct {
values []string
buffer []byte
maxLen int
}
func newStreamRedactor(values []string) streamRedactor {
unique := make(map[string]struct{}, len(values))
for _, value := range values {
if value != "" {
unique[value] = struct{}{}
}
}
sorted := make([]string, 0, len(unique))
for value := range unique {
sorted = append(sorted, value)
}
sort.Slice(sorted, func(i, j int) bool { return len(sorted[i]) > len(sorted[j]) })
maxLen := 1
for _, value := range sorted {
if len(value) > maxLen {
maxLen = len(value)
}
}
return streamRedactor{values: sorted, maxLen: maxLen}
}
func (r *streamRedactor) Write(data []byte) []byte {
r.buffer = append(r.buffer, data...)
safeCut := len(r.buffer) - r.maxLen + 1
if safeCut <= 0 {
return nil
}
emitCut := safeCut
for _, value := range r.values {
start := 0
for {
index := bytes.Index(r.buffer[start:], []byte(value))
if index < 0 {
break
}
index += start
if index+len(value) > safeCut && index < emitCut {
emitCut = index
}
start = index + 1
}
}
output := redactBytes(r.buffer[:emitCut], r.values)
r.buffer = append(r.buffer[:0], r.buffer[emitCut:]...)
return output
}
func (r *streamRedactor) Flush() []byte {
output := redactBytes(r.buffer, r.values)
r.buffer = nil
return output
}
func redactBytes(data []byte, values []string) []byte {
out := append([]byte(nil), data...)
for _, value := range values {
out = bytes.ReplaceAll(out, []byte(value), []byte("<redacted>"))
}
return out
}

View File

@@ -0,0 +1,66 @@
package subprocess
import (
"context"
"errors"
"fmt"
"os/exec"
"time"
)
const (
gracefulTerminationWait = 2 * time.Second
forcefulTerminationWait = 2 * time.Second
)
// ownedProcessTree owns every process started by a command invocation.
// Implementations must tolerate a leader that has already exited.
type ownedProcessTree interface {
Start(*exec.Cmd) error
TerminateGracefully() error
TerminateForcefully() error
Dispose() error
}
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
select {
case waitErr = <-waitCh:
return waitErr, nil, nil, nil
case <-ctx.Done():
ctxErr = ctx.Err()
case captureLimit = <-captureLimits:
}
cleanupErr = tree.TerminateGracefully()
gracefulTimer := time.NewTimer(gracefulTerminationWait)
defer gracefulTimer.Stop()
select {
case waitErr = <-waitCh:
// The leader may exit before descendants finish graceful shutdown.
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
return waitErr, ctxErr, captureLimit, cleanupErr
case <-gracefulTimer.C:
}
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
forcefulTimer := time.NewTimer(forcefulTerminationWait)
defer forcefulTimer.Stop()
select {
case waitErr = <-waitCh:
return waitErr, ctxErr, captureLimit, cleanupErr
case <-forcefulTimer.C:
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
}
}
func joinErrors(errs ...error) error {
filtered := make([]error, 0, len(errs))
for _, err := range errs {
if err != nil {
filtered = append(filtered, err)
}
}
return errors.Join(filtered...)
}

View File

@@ -0,0 +1,227 @@
//go:build linux || darwin || windows
package subprocess
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
func TestRunCancellationTerminatesProcessTree(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
resultCh := make(chan runOutcome, 1)
req, sentinelPath := processTreeRequest(t)
go func() {
result, err := Run(ctx, req)
resultCh <- runOutcome{result: result, err: err}
}()
awaitHelperReady(t, req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"])
cancel()
outcome := awaitRunOutcome(t, resultCh)
if !outcome.result.Canceled {
t.Fatalf("Canceled = %v, want true", outcome.result.Canceled)
}
if !errors.Is(outcome.err, context.Canceled) {
t.Fatalf("error = %v, want context cancellation", outcome.err)
}
assertDescendantDidNotSurvive(t, sentinelPath)
}
func TestRunTimeoutTerminatesProcessTree(t *testing.T) {
req, sentinelPath := processTreeRequest(t)
req.Timeout = 100 * time.Millisecond
result, err := Run(context.Background(), req)
if !result.TimedOut {
t.Fatalf("TimedOut = %v, want true", result.TimedOut)
}
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("error = %v, want context deadline exceeded", err)
}
assertDescendantDidNotSurvive(t, sentinelPath)
}
func TestRunCaptureLimitTerminatesProcessTree(t *testing.T) {
req, sentinelPath := processTreeRequest(t)
req.Args[len(req.Args)-1] = "tree-spam"
result, err := Run(context.Background(), req)
if err == nil {
t.Fatal("Run() error = nil, want capture-limit error")
}
if result.ExitCode == 0 {
t.Fatalf("ExitCode = %d, want terminated process", result.ExitCode)
}
if !strings.Contains(err.Error(), "stdout diagnostic capture for subprocess exceeded") {
t.Fatalf("error = %q, want stdout capture-limit context", err)
}
info, statErr := os.Stat(req.StdoutLogPath)
if statErr != nil {
t.Fatalf("stat stdout diagnostic: %v", statErr)
}
if info.Size() != MaxStdoutDiagnosticBytes {
t.Fatalf("stdout diagnostic size = %d, want %d", info.Size(), MaxStdoutDiagnosticBytes)
}
assertDescendantDidNotSurvive(t, sentinelPath)
}
func TestRunDisposesDescendantsAfterLeaderExit(t *testing.T) {
tests := []struct {
name string
mode string
wantExitCode int
wantWaitDelay bool
ignoreTerm bool
}{
{name: "success retaining streams", mode: "leader-exit-retained", wantExitCode: 0, wantWaitDelay: true},
{name: "success redirecting streams", mode: "leader-exit-redirected", wantExitCode: 0},
{name: "failed leader", mode: "leader-fail-redirected", wantExitCode: 9, ignoreTerm: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, sentinelPath, releasePath := leaderExitRequest(t, tt.mode)
if tt.ignoreTerm {
req.EnvOverrides["SUBPROCESS_HELPER_IGNORE_TERM"] = "1"
}
outcomes := make(chan runOutcome, 1)
go func() {
result, err := Run(context.Background(), req)
outcomes <- runOutcome{result: result, err: err}
}()
var outcome runOutcome
select {
case outcome = <-outcomes:
case <-time.After(6 * time.Second):
t.Fatal("Run() did not complete bounded owned-tree disposal")
}
if outcome.result.ExitCode != tt.wantExitCode {
t.Fatalf("ExitCode = %d, want %d", outcome.result.ExitCode, tt.wantExitCode)
}
if tt.wantWaitDelay {
if !errors.Is(outcome.err, exec.ErrWaitDelay) {
t.Fatalf("error = %v, want exec.ErrWaitDelay", outcome.err)
}
} else if tt.wantExitCode == 0 && outcome.err != nil {
t.Fatalf("Run() error = %v, want nil", outcome.err)
} else if tt.wantExitCode != 0 {
var exitErr *exec.ExitError
if !errors.As(outcome.err, &exitErr) || exitErr.ExitCode() != tt.wantExitCode {
t.Fatalf("error = %v, want exit code %d", outcome.err, tt.wantExitCode)
}
}
if _, err := os.Stat(req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"]); err != nil {
t.Fatalf("descendant readiness file: %v", err)
}
if err := os.WriteFile(releasePath, []byte("release"), 0o600); err != nil {
t.Fatalf("WriteFile(release) error = %v", err)
}
assertDescendantDidNotSurvive(t, sentinelPath)
})
}
}
type runOutcome struct {
result RunResult
err error
}
func processTreeRequest(t *testing.T) (RunRequest, string) {
t.Helper()
executable, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
dir := t.TempDir()
readyPath := filepath.Join(dir, "ready")
sentinelPath := filepath.Join(dir, "descendant-survived")
return RunRequest{
Executable: executable,
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "tree"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"SUBPROCESS_HELPER_READY_PATH": readyPath,
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
},
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: filepath.Join(dir, "stderr.log"),
}, sentinelPath
}
func leaderExitRequest(t *testing.T, mode string) (RunRequest, string, string) {
t.Helper()
executable, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
dir := t.TempDir()
readyPath := filepath.Join(dir, "ready")
releasePath := filepath.Join(dir, "release")
sentinelPath := filepath.Join(dir, "descendant-survived")
return RunRequest{
Executable: executable,
Args: []string{"-test.run=^TestSubprocessHelper$", "--", mode},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"SUBPROCESS_HELPER_READY_PATH": readyPath,
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
},
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: filepath.Join(dir, "stderr.log"),
}, sentinelPath, releasePath
}
func awaitHelperReady(t *testing.T, readyPath string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if _, err := os.Stat(readyPath); err == nil {
return
} else if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("stat helper readiness: %v", err)
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("helper did not start its descendant")
}
func awaitRunOutcome(t *testing.T, outcomes <-chan runOutcome) runOutcome {
t.Helper()
select {
case outcome := <-outcomes:
if outcome.err == nil {
t.Fatal("Run() error = nil, want cancellation error")
}
return outcome
case <-time.After(3 * time.Second):
t.Fatal("Run() did not return after cancellation")
return runOutcome{}
}
}
func assertDescendantDidNotSurvive(t *testing.T, sentinelPath string) {
t.Helper()
time.Sleep(700 * time.Millisecond)
if _, err := os.Stat(sentinelPath); err == nil {
t.Fatal("descendant survived cancellation and wrote its sentinel")
} else if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("stat descendant sentinel: %v", err)
}
}

View File

@@ -0,0 +1,104 @@
//go:build linux || darwin
package subprocess
import (
"errors"
"fmt"
"os"
"os/exec"
"syscall"
"time"
)
const processGroupPollInterval = 10 * time.Millisecond
type unixProcessTree struct {
processGroupID int
}
func newOwnedProcessTree() (ownedProcessTree, error) {
return &unixProcessTree{}, nil
}
func (tree *unixProcessTree) Start(cmd *exec.Cmd) error {
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := cmd.Start(); err != nil {
return err
}
tree.processGroupID = cmd.Process.Pid
return nil
}
func (tree *unixProcessTree) TerminateGracefully() error {
return tree.signal(syscall.SIGTERM)
}
func (tree *unixProcessTree) TerminateForcefully() error {
return tree.signal(syscall.SIGKILL)
}
func (tree *unixProcessTree) Dispose() error {
hasMembers, err := tree.hasMembers()
if err != nil || !hasMembers {
return err
}
cleanupErr := tree.TerminateGracefully()
empty, waitErr := tree.waitUntilEmpty(gracefulTerminationWait)
cleanupErr = joinErrors(cleanupErr, waitErr)
if empty {
return cleanupErr
}
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
empty, waitErr = tree.waitUntilEmpty(forcefulTerminationWait)
cleanupErr = joinErrors(cleanupErr, waitErr)
if !empty {
cleanupErr = joinErrors(cleanupErr, fmt.Errorf("owned subprocess group did not exit within %s after forceful termination", forcefulTerminationWait))
}
return cleanupErr
}
func (tree *unixProcessTree) signal(signal syscall.Signal) error {
if tree.processGroupID <= 0 {
return nil
}
err := syscall.Kill(-tree.processGroupID, signal)
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
return nil
}
return err
}
func (tree *unixProcessTree) hasMembers() (bool, error) {
if tree.processGroupID <= 0 {
return false, nil
}
err := syscall.Kill(-tree.processGroupID, 0)
if err == nil || errors.Is(err, syscall.EPERM) {
return true, nil
}
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
return false, nil
}
return false, fmt.Errorf("inspect owned subprocess group: %w", err)
}
func (tree *unixProcessTree) waitUntilEmpty(timeout time.Duration) (bool, error) {
deadline := time.Now().Add(timeout)
for {
hasMembers, err := tree.hasMembers()
if err != nil || !hasMembers {
return !hasMembers, err
}
remaining := time.Until(deadline)
if remaining <= 0 {
return false, nil
}
if remaining > processGroupPollInterval {
remaining = processGroupPollInterval
}
time.Sleep(remaining)
}
}

View File

@@ -0,0 +1,12 @@
//go:build !linux && !darwin && !windows
package subprocess
import (
"fmt"
"runtime"
)
func newOwnedProcessTree() (ownedProcessTree, error) {
return nil, fmt.Errorf("owned subprocess trees are unsupported on %s", runtime.GOOS)
}

View File

@@ -0,0 +1,122 @@
//go:build windows
package subprocess
import (
"errors"
"fmt"
"os/exec"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
type windowsProcessTree struct {
job windows.Handle
}
func newOwnedProcessTree() (ownedProcessTree, error) {
return &windowsProcessTree{}, nil
}
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
job, err := windows.CreateJobObject(nil, nil)
if err != nil {
return fmt.Errorf("create job object: %w", err)
}
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
_ = windows.CloseHandle(job)
return fmt.Errorf("configure job object: %w", err)
}
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
if err := cmd.Start(); err != nil {
_ = windows.CloseHandle(job)
return err
}
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
if err == nil {
err = windows.AssignProcessToJobObject(job, process)
_ = windows.CloseHandle(process)
}
if err == nil {
err = resumeInitialThread(uint32(cmd.Process.Pid))
}
if err != nil {
killErr := cmd.Process.Kill()
waitErr := cmd.Wait()
_ = windows.CloseHandle(job)
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
}
tree.job = job
return nil
}
func resumeInitialThread(processID uint32) error {
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
if err != nil {
return fmt.Errorf("snapshot initial thread: %w", err)
}
defer func() { _ = windows.CloseHandle(snapshot) }()
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
if err := windows.Thread32First(snapshot, &entry); err != nil {
return fmt.Errorf("find initial thread: %w", err)
}
for {
if entry.OwnerProcessID != processID {
// Keep enumerating until the suspended process's only initial thread
// is found.
} else {
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
if openErr != nil {
return fmt.Errorf("open initial thread: %w", openErr)
}
defer func() { _ = windows.CloseHandle(thread) }()
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
return fmt.Errorf("resume initial thread: %w", resumeErr)
}
return nil
}
if err := windows.Thread32Next(snapshot, &entry); err != nil {
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
break
}
return fmt.Errorf("find initial thread: %w", err)
}
}
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
}
func (tree *windowsProcessTree) TerminateGracefully() error {
// Windows jobs have no portable graceful signal. Terminating the owned job
// is the safe fallback and prevents a descendant from escaping cleanup.
return tree.terminate()
}
func (tree *windowsProcessTree) TerminateForcefully() error {
return tree.terminate()
}
func (tree *windowsProcessTree) Dispose() error {
if tree.job == 0 {
return nil
}
err := windows.CloseHandle(tree.job)
tree.job = 0
return err
}
func (tree *windowsProcessTree) terminate() error {
if tree.job == 0 {
return nil
}
return windows.TerminateJobObject(tree.job, 1)
}

View File

@@ -2,16 +2,13 @@ package subprocess
import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gopkg.in/yaml.v3"
)
@@ -21,6 +18,8 @@ type RunRequest struct {
Args []string
WorkingDir string
EnvOverrides map[string]string
SensitiveEnvNames []string
DiagnosticOwner string
Timeout time.Duration
StdoutLogPath string
StderrLogPath string
@@ -54,17 +53,26 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
}
defer cancel()
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath)
childEnv := buildChildEnvironment(os.Environ(), req.EnvOverrides)
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath, req.diagnosticOwner(), sensitiveEnvironmentValues(childEnv, req.SensitiveEnvNames))
if err != nil {
return RunResult{}, err
}
defer logs.Close()
cmd := exec.CommandContext(runCtx, req.Executable, req.Args...)
tree, err := newOwnedProcessTree()
if err != nil {
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
}
cmd := exec.Command(req.Executable, req.Args...)
cmd.Dir = req.WorkingDir
cmd.Env = mergeEnv(os.Environ(), req.EnvOverrides)
cmd.Env = childEnv
cmd.Stdout = logs.Stdout
cmd.Stderr = logs.Stderr
// Streaming capture uses pipes. Bound their lifetime when a leader exits
// while a descendant still holds a stream descriptor.
cmd.WaitDelay = forcefulTerminationWait
started := time.Now().UTC()
result := RunResult{
@@ -74,45 +82,67 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
StderrLogPath: req.StderrLogPath,
}
if err := cmd.Start(); err != nil {
if err := runCtx.Err(); err != nil {
result.CompletedAt = time.Now().UTC()
result.Duration = result.CompletedAt.Sub(result.StartedAt)
return result, fmt.Errorf("command was not started: %w", err)
}
if err := tree.Start(cmd); err != nil {
result.CompletedAt = time.Now().UTC()
result.Duration = result.CompletedAt.Sub(result.StartedAt)
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
}
waitErr := cmd.Wait()
waitCh := make(chan error, 1)
go func() { waitCh <- cmd.Wait() }()
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
cleanupErr = joinErrors(cleanupErr, logs.Flush())
if captureLimit == nil {
captureLimit = logs.Limit()
}
result.CompletedAt = time.Now().UTC()
result.Duration = result.CompletedAt.Sub(result.StartedAt)
if cmd.ProcessState != nil {
result.ExitCode = cmd.ProcessState.ExitCode()
}
ctxErr := runCtx.Err()
if errors.Is(ctxErr, context.DeadlineExceeded) {
if ctxErr == context.DeadlineExceeded {
result.TimedOut = true
}
if errors.Is(ctxErr, context.Canceled) && !result.TimedOut {
if ctxErr == context.Canceled && !result.TimedOut {
result.Canceled = true
}
if waitErr == nil {
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
return result, nil
}
stderrTail := readRedactedTail(req.StderrLogPath, req.EnvOverrides, 2048)
stderrTail := logs.stderr.Tail()
diagnostics := buildDiagnostics(req, result, stderrTail)
if captureLimit != nil {
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
}
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
}
if result.TimedOut {
return result, fmt.Errorf("command timed out after %s (%s)", req.Timeout, diagnostics)
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
}
if result.Canceled {
return result, fmt.Errorf("command canceled (%s)", diagnostics)
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
}
if exitErr, ok := waitErr.(*exec.ExitError); ok {
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, waitErr)
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
}
if cleanupErr != nil {
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
}
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, waitErr)
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
}
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
@@ -127,141 +157,17 @@ func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
return nil
}
// WriteFileAtomic writes bytes via same-directory temp file + atomic rename.
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("write file: path is required")
}
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create parent directory %q: %w", dir, err)
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
return fmt.Errorf("write file %q: %w", path, err)
}
base := filepath.Base(path)
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write temp file: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return fmt.Errorf("sync temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close temp file: %w", err)
}
if err := os.Chmod(tmpPath, perm); err != nil {
return fmt.Errorf("chmod temp file: %w", err)
}
if err := os.Rename(tmpPath, path); err != nil {
return fmt.Errorf("rename temp file: %w", err)
}
removeTmp = false
return nil
}
type logWriters struct {
files []*os.File
Stdout io.Writer
Stderr io.Writer
}
func (l *logWriters) Close() {
for _, f := range l.files {
_ = f.Close()
}
}
func openLogWriters(stdoutPath, stderrPath string) (*logWriters, error) {
cleanStdout := cleanLogPath(stdoutPath)
cleanStderr := cleanLogPath(stderrPath)
// Keep stdout/stderr on the same file descriptor when both paths target
// the same file to avoid descriptor aliasing surprises across runtimes.
if cleanStdout != "" && cleanStdout == cleanStderr {
f, err := openLogFile(cleanStdout)
if err != nil {
return nil, fmt.Errorf("open shared stdout/stderr log %q: %w", cleanStdout, err)
}
return &logWriters{
files: []*os.File{f},
Stdout: f,
Stderr: f,
}, nil
}
stdoutFile, stdoutWriter, err := logWriter(cleanStdout)
if err != nil {
return nil, fmt.Errorf("open stdout log: %w", err)
}
stderrFile, stderrWriter, err := logWriter(cleanStderr)
if err != nil {
closeFile(stdoutFile)
return nil, fmt.Errorf("open stderr log: %w", err)
}
files := make([]*os.File, 0, 2)
if stdoutFile != nil {
files = append(files, stdoutFile)
}
if stderrFile != nil {
files = append(files, stderrFile)
}
return &logWriters{
files: files,
Stdout: stdoutWriter,
Stderr: stderrWriter,
}, nil
}
func cleanLogPath(path string) string {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return ""
}
return filepath.Clean(trimmed)
}
func logWriter(path string) (*os.File, io.Writer, error) {
if strings.TrimSpace(path) == "" {
return nil, io.Discard, nil
}
f, err := openLogFile(path)
if err != nil {
return nil, nil, err
}
return f, f, nil
}
func openLogFile(path string) (*os.File, error) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
}
f, err := os.Create(path)
if err != nil {
return nil, fmt.Errorf("open log file %q: %w", path, err)
}
return f, nil
}
func closeFile(f *os.File) {
if f != nil {
_ = f.Close()
}
}
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
details := fmt.Sprintf(
"executable=%q args=%v cwd=%q timeout=%s exit_code=%d timed_out=%t canceled=%t stdout_log=%q stderr_log=%q",
@@ -298,87 +204,3 @@ func fdDiagnosticsHint(exitCode int, stderrTail string) string {
}
return ""
}
func readRedactedTail(path string, envOverrides map[string]string, maxBytes int64) string {
if strings.TrimSpace(path) == "" || maxBytes <= 0 {
return ""
}
f, err := os.Open(path)
if err != nil {
return ""
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return ""
}
size := info.Size()
start := int64(0)
if size > maxBytes {
start = size - maxBytes
}
if _, err := f.Seek(start, io.SeekStart); err != nil {
return ""
}
data, err := io.ReadAll(f)
if err != nil {
return ""
}
tail := strings.TrimSpace(string(data))
if tail == "" {
return ""
}
return redactSensitiveTail(tail, envOverrides)
}
func redactSensitiveTail(tail string, envOverrides map[string]string) string {
out := tail
for k, v := range envOverrides {
if strings.TrimSpace(v) == "" {
continue
}
if looksSensitiveEnvKey(k) {
out = strings.ReplaceAll(out, v, "<redacted>")
}
}
return out
}
func looksSensitiveEnvKey(key string) bool {
k := strings.ToUpper(strings.TrimSpace(key))
return strings.Contains(k, "KEY") ||
strings.Contains(k, "TOKEN") ||
strings.Contains(k, "SECRET") ||
strings.Contains(k, "PASSWORD")
}
func mergeEnv(base []string, overrides map[string]string) []string {
if len(overrides) == 0 {
return base
}
kv := make(map[string]string, len(base)+len(overrides))
for _, item := range base {
k, v, ok := strings.Cut(item, "=")
if !ok {
continue
}
kv[k] = v
}
for k, v := range overrides {
kv[k] = v
}
keys := make([]string, 0, len(kv))
for k := range kv {
keys = append(keys, k)
}
sort.Strings(keys)
out := make([]string, 0, len(keys))
for _, k := range keys {
out = append(out, k+"="+kv[k])
}
return out
}

View File

@@ -1,10 +1,17 @@
package subprocess
import (
"bytes"
"context"
"errors"
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"strings"
"syscall"
"testing"
"time"
@@ -120,6 +127,316 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
if !strings.Contains(err.Error(), "<redacted>") {
t.Fatalf("error = %q, want redacted stderr tail marker", err.Error())
}
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
data, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(data), secretValue) {
t.Fatalf("diagnostic %q leaked secret: %q", path, data)
}
}
}
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
}
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
dir := t.TempDir()
targetPath := filepath.Join(dir, "outside.log")
const original = "must remain unchanged"
if err := os.WriteFile(targetPath, []byte(original), 0o600); err != nil {
t.Fatalf("WriteFile(target) error = %v", err)
}
stdoutPath := filepath.Join(dir, "stdout.log")
if err := os.Symlink(targetPath, stdoutPath); err != nil {
t.Fatalf("Symlink() error = %v", err)
}
_, err = Run(context.Background(), RunRequest{
Executable: exe,
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "success"},
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1"},
StdoutLogPath: stdoutPath,
StderrLogPath: filepath.Join(dir, "stderr.log"),
})
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
}
data, readErr := os.ReadFile(targetPath)
if readErr != nil {
t.Fatalf("ReadFile(target) error = %v", readErr)
}
if string(data) != original {
t.Fatalf("target content = %q, want %q", data, original)
}
}
func TestRunFailureUsesOpenedDiagnosticAfterPathReplacement(t *testing.T) {
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
dir := t.TempDir()
readyPath := filepath.Join(dir, "ready")
releasePath := filepath.Join(dir, "release")
stderrPath := filepath.Join(dir, "stderr.log")
openedPath := filepath.Join(dir, "opened-stderr.log")
const secretValue = "replacement-api-key-value"
const commandContent = "trusted command failure"
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "delayed-fail"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"API_KEY": secretValue,
"SUBPROCESS_HELPER_READY_PATH": readyPath,
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
"SUBPROCESS_HELPER_STDERR": commandContent,
},
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: stderrPath,
}
resultCh := make(chan error, 1)
go func() {
_, runErr := Run(context.Background(), req)
resultCh <- runErr
}()
waitForHelperFile(t, readyPath)
if err := os.Rename(stderrPath, openedPath); err != nil {
t.Fatalf("Rename(stderr log) error = %v", err)
}
if err := os.WriteFile(stderrPath, []byte(secretValue), 0o600); err != nil {
t.Fatalf("WriteFile(replacement) error = %v", err)
}
if err := os.WriteFile(releasePath, []byte("continue"), 0o600); err != nil {
t.Fatalf("WriteFile(release) error = %v", err)
}
select {
case runErr := <-resultCh:
if runErr == nil {
t.Fatal("Run() error = nil, want command failure")
}
if strings.Contains(runErr.Error(), secretValue) {
t.Fatalf("error read replacement-path content: %q", runErr)
}
if !strings.Contains(runErr.Error(), commandContent) {
t.Fatalf("error = %q, want retained command diagnostic", runErr)
}
case <-time.After(3 * time.Second):
t.Fatal("Run() did not return after helper release")
}
openedData, err := os.ReadFile(openedPath)
if err != nil {
t.Fatalf("ReadFile(opened diagnostic) error = %v", err)
}
if !strings.Contains(string(openedData), commandContent) {
t.Fatalf("opened diagnostic = %q, want command content", openedData)
}
replacementData, err := os.ReadFile(stderrPath)
if err != nil {
t.Fatalf("ReadFile(replacement diagnostic) error = %v", err)
}
if string(replacementData) != secretValue {
t.Fatalf("replacement diagnostic = %q, want %q", replacementData, secretValue)
}
}
func TestRunRedactsSplitCredentialInSeparateAndSharedDiagnostics(t *testing.T) {
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
const secretValue = "split-super-secret-value"
for _, shared := range []bool{false, true} {
t.Run(map[bool]string{false: "separate", true: "shared"}[shared], func(t *testing.T) {
dir := t.TempDir()
stdoutPath := filepath.Join(dir, "stdout.log")
stderrPath := filepath.Join(dir, "stderr.log")
if shared {
stderrPath = stdoutPath
}
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "splitsecret"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"API_KEY": secretValue,
},
StdoutLogPath: stdoutPath,
StderrLogPath: stderrPath,
}
_, runErr := Run(context.Background(), req)
if runErr == nil {
t.Fatal("Run() error = nil, want command failure")
}
if strings.Contains(runErr.Error(), secretValue) || !strings.Contains(runErr.Error(), "<redacted>") {
t.Fatalf("error = %q, want redacted credential", runErr)
}
paths := map[string]struct{}{stdoutPath: {}, stderrPath: {}}
for path := range paths {
data, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("ReadFile(%q) error = %v", path, readErr)
}
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
t.Fatalf("diagnostic %q = %q, want redacted credential", path, data)
}
}
})
}
}
func TestRunRedactsInheritedSensitiveEnvironment(t *testing.T) {
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
secretValue := "inherited-secret-value"
t.Setenv("OPENROUTER_API_KEY", secretValue)
dir := t.TempDir()
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
},
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: filepath.Join(dir, "stderr.log"),
}
_, err = Run(context.Background(), req)
if err == nil {
t.Fatal("Run() error = nil, want non-nil")
}
if strings.Contains(err.Error(), secretValue) {
t.Fatalf("error leaked inherited secret: %q", err)
}
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
data, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(data), secretValue) {
t.Fatalf("diagnostic %q leaked inherited secret: %q", path, data)
}
}
}
func TestRunRedactsSensitiveOutputAndErrorTail(t *testing.T) {
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
}
secretValue := "override-secret-value"
dir := t.TempDir()
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
"OPENROUTER_API_KEY": secretValue,
},
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: filepath.Join(dir, "stderr.log"),
}
_, err = Run(context.Background(), req)
if err == nil {
t.Fatal("Run() error = nil, want non-nil")
}
if strings.Contains(err.Error(), secretValue) || !strings.Contains(err.Error(), "<redacted>") {
t.Fatalf("error = %q, want redacted secret", err)
}
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
data, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
t.Fatalf("diagnostic %q = %q, want redacted secret", path, data)
}
}
}
func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
redactor := newStreamRedactor([]string{"abc", "abcde", "cde", ""})
var output bytes.Buffer
output.Write(redactor.Write([]byte("start-ab")))
output.Write(redactor.Write([]byte("cde-end")))
output.Write(redactor.Flush())
if got := output.String(); got != "start-<redacted>-end" {
t.Fatalf("redacted output = %q, want one redacted marker", got)
}
}
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
var exactOutput bytes.Buffer
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
if _, err := exact.Write([]byte("abcde")); err != nil {
t.Fatalf("exact Write() error = %v", err)
}
if err := exact.Flush(); err != nil {
t.Fatalf("exact Flush() error = %v", err)
}
if got := exactOutput.String(); got != "abcde" {
t.Fatalf("exact output = %q, want abcde", got)
}
if exactLogs.Limit() != nil {
t.Fatal("exact write recorded a capture limit")
}
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
var cappedOutput bytes.Buffer
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
if _, err := capped.Write([]byte("abcdef")); err == nil {
t.Fatal("cap-plus-one Write() error = nil, want capture limit")
}
if err := capped.Flush(); err != nil {
t.Fatalf("cap-plus-one Flush() error = %v", err)
}
if got := cappedOutput.String(); got != "abcde" {
t.Fatalf("capped output = %q, want abcde", got)
}
if limit := cappedLogs.Limit(); limit == nil || limit.stream != "stderr" || limit.limit != 5 {
t.Fatalf("capture limit = %#v, want stderr limit 5", limit)
}
}
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
var output bytes.Buffer
secret := "credential-value"
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
prefix := strings.Repeat("x", diagnosticTailBytes+512)
if _, err := writer.Write([]byte(prefix + secret[:7])); err != nil {
t.Fatalf("first Write() error = %v", err)
}
if _, err := writer.Write([]byte(secret[7:] + "-failure")); err != nil {
t.Fatalf("second Write() error = %v", err)
}
if err := writer.Flush(); err != nil {
t.Fatalf("Flush() error = %v", err)
}
tail := writer.Tail()
if len(tail) > diagnosticTailBytes {
t.Fatalf("retained tail length = %d, want at most %d", len(tail), diagnosticTailBytes)
}
if strings.Contains(tail, secret) || !strings.Contains(tail, "<redacted>-failure") {
t.Fatalf("retained tail = %q, want bounded redacted content", tail)
}
}
func TestRunFailureAddsBadDescriptorHint(t *testing.T) {
@@ -184,15 +501,14 @@ func TestRunInheritsParentEnvironmentByDefault(t *testing.T) {
t.Fatalf("os.Executable() error = %v", err)
}
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
t.Setenv("SUBPROCESS_PARENT_VALUE", "inherited-value")
t.Setenv("PATH", "inherited-value")
dir := t.TempDir()
stdoutPath := filepath.Join(dir, "stdout.log")
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1", "SUBPROCESS_HELPER_ENV_KEY": "PATH"},
StdoutLogPath: stdoutPath,
}
@@ -214,16 +530,16 @@ func TestRunEnvOverridesWinOverInheritedValues(t *testing.T) {
t.Fatalf("os.Executable() error = %v", err)
}
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
t.Setenv("SUBPROCESS_PARENT_VALUE", "parent-value")
dir := t.TempDir()
stdoutPath := filepath.Join(dir, "stdout.log")
req := RunRequest{
Executable: exe,
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
EnvOverrides: map[string]string{"SUBPROCESS_PARENT_VALUE": "override-value"},
EnvOverrides: map[string]string{
"GO_WANT_SUBPROCESS_HELPER": "1",
"SUBPROCESS_HELPER_ENV_KEY": "SUBPROCESS_PARENT_VALUE",
"SUBPROCESS_PARENT_VALUE": "override-value",
},
StdoutLogPath: stdoutPath,
}
@@ -361,6 +677,30 @@ func TestSubprocessHelper(t *testing.T) {
case "failbadfd":
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
os.Exit(120)
case "delayed-fail":
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
os.Exit(4)
}
deadline := time.Now().Add(2 * time.Second)
for {
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
break
} else if !errors.Is(err, os.ErrNotExist) || time.Now().After(deadline) {
os.Exit(5)
}
time.Sleep(10 * time.Millisecond)
}
_, _ = os.Stderr.WriteString(os.Getenv("SUBPROCESS_HELPER_STDERR"))
os.Exit(6)
case "splitsecret":
secret := os.Getenv("API_KEY")
split := len(secret) / 2
for _, stream := range []*os.File{os.Stdout, os.Stderr} {
_, _ = stream.WriteString(secret[:split])
time.Sleep(20 * time.Millisecond)
_, _ = stream.WriteString(secret[split:] + "\n")
}
os.Exit(7)
case "sleep":
time.Sleep(500 * time.Millisecond)
os.Exit(0)
@@ -368,7 +708,115 @@ func TestSubprocessHelper(t *testing.T) {
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
_, _ = os.Stdout.WriteString(os.Getenv(key) + "\n")
os.Exit(0)
case "echoenv":
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
value := os.Getenv(key)
_, _ = os.Stdout.WriteString(value)
_, _ = os.Stderr.WriteString(value)
os.Exit(5)
case "spam":
chunk := strings.Repeat("x", 64*1024)
count, _ := strconv.Atoi(os.Getenv("SUBPROCESS_HELPER_CHUNKS"))
for range count {
_, _ = os.Stdout.WriteString(chunk)
}
os.Exit(0)
case "tree-spam":
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
descendant.Stdout = os.Stdout
descendant.Stderr = os.Stderr
if err := descendant.Start(); err != nil {
os.Exit(3)
}
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
os.Exit(4)
}
chunk := strings.Repeat("x", 64*1024)
for {
_, _ = os.Stdout.WriteString(chunk)
}
case "tree":
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
descendant.Stdout = os.Stdout
descendant.Stderr = os.Stderr
if err := descendant.Start(); err != nil {
os.Exit(3)
}
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
os.Exit(4)
}
time.Sleep(10 * time.Second)
os.Exit(0)
case "leader-exit-retained", "leader-exit-redirected", "leader-fail-redirected":
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant-after-release")
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
if mode == "leader-exit-retained" {
descendant.Stdout = os.Stdout
descendant.Stderr = os.Stderr
}
if err := descendant.Start(); err != nil {
os.Exit(3)
}
if !helperFileAppeared(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), 2*time.Second) {
os.Exit(4)
}
if mode == "leader-fail-redirected" {
os.Exit(9)
}
os.Exit(0)
case "descendant-after-release":
if os.Getenv("SUBPROCESS_HELPER_IGNORE_TERM") == "1" {
signal.Ignore(syscall.SIGTERM)
}
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
os.Exit(4)
}
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
os.Exit(0)
} else if !errors.Is(err, os.ErrNotExist) {
os.Exit(5)
}
time.Sleep(10 * time.Millisecond)
}
os.Exit(0)
case "descendant":
time.Sleep(500 * time.Millisecond)
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
time.Sleep(10 * time.Second)
os.Exit(0)
default:
os.Exit(2)
}
}
func waitForHelperFile(t *testing.T, path string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if _, err := os.Stat(path); err == nil {
return
} else if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("Stat(%q) error = %v", path, err)
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("helper file %q was not created", path)
}
func helperFileAppeared(path string, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if _, err := os.Stat(path); err == nil {
return true
} else if !errors.Is(err, os.ErrNotExist) {
return false
}
time.Sleep(10 * time.Millisecond)
}
return false
}

View File

@@ -2,8 +2,10 @@ package whisperx
import (
"context"
"os"
"path/filepath"
"sync"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
var minimalTranscriptJSON = []byte(`{"schema":"speaker_transcript.v1","segments":[]}`)
@@ -31,7 +33,8 @@ func (n *NoopClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
// FakeClient captures requests and returns deterministic responses for tests.
type FakeClient struct {
Requests []TranscribeRequest
requestsMu sync.RWMutex
requests []TranscribeRequest
Err error
Result TranscribeResult
TranscribeFn func(ctx context.Context, req TranscribeRequest) (TranscribeResult, error)
@@ -42,7 +45,9 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
if err := ctx.Err(); err != nil {
return TranscribeResult{}, err
}
f.Requests = append(f.Requests, req)
f.requestsMu.Lock()
f.requests = append(f.requests, req)
f.requestsMu.Unlock()
if f.TranscribeFn != nil {
return f.TranscribeFn(ctx, req)
}
@@ -65,12 +70,19 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
return res, nil
}
// RequestsSnapshot returns a copy of captured requests safe for concurrent test assertions.
func (f *FakeClient) RequestsSnapshot() []TranscribeRequest {
f.requestsMu.RLock()
defer f.requestsMu.RUnlock()
return append([]TranscribeRequest(nil), f.requests...)
}
func writeMinimalJSON(path string) error {
if path == "" {
return nil
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
return err
}
return os.WriteFile(path, minimalTranscriptJSON, 0o644)
return fileops.WriteFileAtomic(path, minimalTranscriptJSON, fileops.WorkspaceFileMode)
}

View File

@@ -3,6 +3,7 @@ package whisperx
import (
"context"
"errors"
"sync"
"testing"
)
@@ -14,8 +15,9 @@ func TestFakeClientCapturesRequestAndReturnsPath(t *testing.T) {
if err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
if len(fake.Requests) != 1 || fake.Requests[0].SpeakerID != "alice" {
t.Fatalf("requests = %#v, want one alice request", fake.Requests)
requests := fake.RequestsSnapshot()
if len(requests) != 1 || requests[0].SpeakerID != "alice" {
t.Fatalf("requests = %#v, want one alice request", requests)
}
if res.OutputRawTranscriptPath != req.OutputRawTranscriptPath {
t.Fatalf("output path = %q, want %q", res.OutputRawTranscriptPath, req.OutputRawTranscriptPath)
@@ -29,3 +31,22 @@ func TestFakeClientError(t *testing.T) {
t.Fatal("expected error, got nil")
}
}
func TestFakeClientRequestsSnapshotSupportsConcurrentCalls(t *testing.T) {
fake := &FakeClient{}
const callers = 16
var group sync.WaitGroup
group.Add(callers)
for i := 0; i < callers; i++ {
go func() {
defer group.Done()
if _, err := fake.Transcribe(context.Background(), TranscribeRequest{}); err != nil {
t.Errorf("Transcribe() error = %v", err)
}
}()
}
group.Wait()
if got := len(fake.RequestsSnapshot()); got != callers {
t.Fatalf("captured requests = %d, want %d", got, callers)
}
}

View File

@@ -1,7 +1,6 @@
package whisperx
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -14,10 +13,16 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
const defaultMaxResponseBytes int64 = 10 * 1024 * 1024
const (
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
whisperXUploadBufferSize = 32 * 1024
)
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
type HTTPClientConfig struct {
@@ -39,6 +44,7 @@ type HTTPClient struct {
retryDelay time.Duration
httpClient *http.Client
maxResponseBytes int64
openAudio func(string) (io.ReadCloser, error)
}
// NewHTTPClientFromConfigValues builds a client from config values and parses durations once.
@@ -72,11 +78,11 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
return nil, fmt.Errorf("whisperx transcribe_url is required")
}
u, err := url.Parse(cfg.TranscribeURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil || !u.IsAbs() || u.Host == "" || !isHTTPURLScheme(u.Scheme) {
if err != nil {
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: %w", cfg.TranscribeURL, err)
}
return nil, fmt.Errorf("invalid whisperx transcribe_url %q", cfg.TranscribeURL)
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: must be an absolute http or https URL", cfg.TranscribeURL)
}
if cfg.Timeout <= 0 {
return nil, fmt.Errorf("whisperx timeout must be > 0")
@@ -93,12 +99,14 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
client := cfg.HTTPClient
if client == nil {
client = &http.Client{}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ExpectContinueTimeout = 100 * time.Millisecond
client = &http.Client{Transport: transport}
}
maxBytes := cfg.MaxResponseBytes
if maxBytes <= 0 {
maxBytes = defaultMaxResponseBytes
maxBytes = defaultMaxWhisperXResponseBytes
}
return &HTTPClient{
@@ -109,6 +117,7 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
retryDelay: cfg.RetryDelay,
httpClient: client,
maxResponseBytes: maxBytes,
openAudio: func(path string) (io.ReadCloser, error) { return os.Open(path) },
}, nil
}
@@ -147,7 +156,7 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
result.Duration = time.Since(start)
return result, fmt.Errorf("whisperx attempt %d returned invalid json: %w", attempt, err)
}
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, 0o644); err != nil {
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, fileops.WorkspaceFileMode); err != nil {
result.Duration = time.Since(start)
return result, fmt.Errorf("whisperx write transcript output %q: %w", req.OutputRawTranscriptPath, err)
}
@@ -183,53 +192,45 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
}
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
bodyBuf := &bytes.Buffer{}
writer := multipart.NewWriter(bodyBuf)
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
defer upload.Close()
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
if err != nil {
return 0, nil, fmt.Errorf("create multipart file field: %w", err)
}
audioFile, err := os.Open(audioPath)
if err != nil {
return 0, nil, fmt.Errorf("open audio file %q: %w", audioPath, err)
}
if _, err := io.Copy(fileWriter, audioFile); err != nil {
_ = audioFile.Close()
return 0, nil, fmt.Errorf("copy audio file %q: %w", audioPath, err)
}
if err := audioFile.Close(); err != nil {
return 0, nil, fmt.Errorf("close audio file %q: %w", audioPath, err)
}
if err := writer.WriteField("language", c.language); err != nil {
return 0, nil, fmt.Errorf("write language form field: %w", err)
}
if err := writer.Close(); err != nil {
return 0, nil, fmt.Errorf("close multipart writer: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), bodyBuf)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
if err != nil {
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("Content-Type", upload.contentType)
req.Header.Set("Expect", "100-continue")
resp, err := c.httpClient.Do(req)
if err != nil {
_ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil {
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
}
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
}
defer resp.Body.Close()
data, err := readBounded(resp.Body, c.maxResponseBytes)
if err != nil {
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
_ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
}
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
}
if err := upload.Wait(); err != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
}
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
if err != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
}
return resp.StatusCode, data, nil
}
@@ -264,57 +265,188 @@ func (c *HTTPClient) shouldRetry(parent context.Context, err error, status int)
return false
}
func readBounded(r io.Reader, maxBytes int64) ([]byte, error) {
func readWhisperXResponse(r io.Reader, maxBytes int64) ([]byte, error) {
limited := io.LimitReader(r, maxBytes+1)
data, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if int64(len(data)) > maxBytes {
return nil, fmt.Errorf("response exceeds max size %d bytes", maxBytes)
return nil, fmt.Errorf("whisperx response exceeds configured limit of %d bytes", maxBytes)
}
return data, nil
}
func isHTTPURLScheme(scheme string) bool {
switch strings.ToLower(scheme) {
case "http", "https":
return true
default:
return false
}
}
type multipartUpload struct {
reader *io.PipeReader
writer *io.PipeWriter
contentType string
done chan struct{}
mu sync.Mutex
audio io.Closer
err error
aborted bool
}
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
reader, writer := io.Pipe()
multipartWriter := multipart.NewWriter(writer)
upload := &multipartUpload{
reader: reader,
writer: writer,
contentType: multipartWriter.FormDataContentType(),
done: make(chan struct{}),
}
go func() {
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
if err != nil {
_ = writer.CloseWithError(err)
} else {
_ = writer.Close()
}
upload.mu.Lock()
upload.err = err
upload.audio = nil
upload.mu.Unlock()
close(upload.done)
}()
go func() {
select {
case <-ctx.Done():
upload.abort()
case <-upload.done:
}
}()
return upload
}
func (u *multipartUpload) Read(p []byte) (int, error) {
return u.reader.Read(p)
}
func (u *multipartUpload) Close() error {
u.abort()
return nil
}
func (u *multipartUpload) Wait() error {
<-u.done
u.mu.Lock()
defer u.mu.Unlock()
return u.err
}
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
if err != nil {
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
}
audioFile, err := openAudio(audioPath)
if err != nil {
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
}
u.setAudio(audioFile)
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
closeErr := audioFile.Close()
u.clearAudio(audioFile)
if copyErr != nil {
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
}
if closeErr != nil {
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
}
if err := writer.WriteField("language", language); err != nil {
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
}
if err := writer.Close(); err != nil {
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
}
return nil
}
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
if ctx.Err() != nil {
return ctx.Err()
}
u.mu.Lock()
aborted := u.aborted
u.mu.Unlock()
if aborted {
return nil
}
return err
}
func (u *multipartUpload) setAudio(audio io.Closer) {
u.mu.Lock()
u.audio = audio
aborted := u.aborted
u.mu.Unlock()
if aborted {
_ = audio.Close()
}
}
func (u *multipartUpload) clearAudio(audio io.Closer) {
u.mu.Lock()
if u.audio == audio {
u.audio = nil
}
u.mu.Unlock()
}
func (u *multipartUpload) abort() {
u.mu.Lock()
if u.aborted {
u.mu.Unlock()
return
}
u.aborted = true
audio := u.audio
u.mu.Unlock()
_ = u.reader.Close()
if audio != nil {
_ = audio.Close()
}
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r *contextReader) Read(p []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.reader.Read(p)
}
}
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("path is required")
}
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create parent dir %q: %w", dir, err)
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
return fmt.Errorf("write file %q: %w", path, err)
}
base := filepath.Base(path)
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
if err != nil {
return fmt.Errorf("create temp file: %w", err)
}
tmpPath := tmp.Name()
removeTmp := true
defer func() {
if removeTmp {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("write temp file: %w", err)
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return fmt.Errorf("sync temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close temp file: %w", err)
}
if err := os.Chmod(tmpPath, perm); err != nil {
return fmt.Errorf("chmod temp file: %w", err)
}
if err := os.Rename(tmpPath, path); err != nil {
return fmt.Errorf("rename temp file: %w", err)
}
removeTmp = false
return nil
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
@@ -19,6 +20,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
var gotLanguage string
var gotFileField string
var gotFileSize int
var gotFileData string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
@@ -40,6 +42,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
t.Fatalf("ReadAll(file) error = %v", err)
}
gotFileSize = len(data)
gotFileData = string(data)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"schema":"speaker_transcript.v1","segments":[]}`))
@@ -77,13 +80,27 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
if gotFileSize == 0 {
t.Fatal("file size = 0, want >0")
}
if gotFileData != "audio-data" {
t.Fatalf("file data = %q, want exact payload", gotFileData)
}
verifyJSONFile(t, outPath)
}
func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
var calls atomic.Int32
var payloads []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := calls.Add(1)
file, _, err := r.FormFile("file")
if err != nil {
t.Fatalf("FormFile(file) error = %v", err)
}
data, err := io.ReadAll(file)
_ = file.Close()
if err != nil {
t.Fatalf("ReadAll(file) error = %v", err)
}
payloads = append(payloads, string(data))
if n == 1 {
http.Error(w, "temporary", http.StatusInternalServerError)
return
@@ -112,6 +129,9 @@ func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
if calls.Load() != 2 {
t.Fatalf("calls = %d, want 2", calls.Load())
}
if len(payloads) != 2 || payloads[0] != "audio-data" || payloads[1] != "audio-data" {
t.Fatalf("retry payloads = %#v, want two exact audio payloads", payloads)
}
verifyJSONFile(t, outPath)
}
@@ -145,7 +165,7 @@ func TestHTTPClientDoesNotRetryOnNonRetryableStatus(t *testing.T) {
}
}
func TestHTTPClientInvalidJSONFailsAndDoesNotPromote(t *testing.T) {
func TestHTTPClientInvalidJSONFailsAndDoesNotInstallOutput(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`not-json`))
}))
@@ -247,7 +267,244 @@ func TestHTTPClientConstructorValidation(t *testing.T) {
if err == nil {
t.Fatal("expected bad retry_delay error")
}
for _, endpoint := range []string{"ftp://example.com/transcribe", "file:///tmp/transcribe", "//example.com/transcribe", "https:/missing-host"} {
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err == nil {
t.Errorf("NewHTTPClientFromConfigValues(%q) error = nil, want endpoint validation error", endpoint)
}
}
for _, endpoint := range []string{"http://example.com/transcribe", "https://example.com/transcribe"} {
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err != nil {
t.Errorf("NewHTTPClientFromConfigValues(%q) error = %v", endpoint, err)
}
}
}
func TestHTTPClientStreamsUploadBeforeSourceCompletes(t *testing.T) {
release := make(chan struct{})
source := newGatedReadCloser([]byte("audio-data"), release)
firstByteReceived := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
part := firstMultipartFilePart(t, r)
buf := make([]byte, 1)
if _, err := part.Read(buf); err != nil {
t.Errorf("Read(file) error = %v", err)
return
}
close(firstByteReceived)
if _, err := io.Copy(io.Discard, part); err != nil {
t.Errorf("discard remaining file data: %v", err)
return
}
_, _ = w.Write([]byte(`{"ok":true}`))
}))
defer srv.Close()
client := newTestHTTPClient(t, srv.URL)
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
done := make(chan error, 1)
go func() {
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
done <- err
}()
select {
case <-firstByteReceived:
close(release)
case <-time.After(time.Second):
t.Fatal("server did not receive streamed audio before source completed")
}
if err := <-done; err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
}
func TestHTTPClientSourceReadFailureReachesCaller(t *testing.T) {
sourceErr := errors.New("source read failed")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
}))
defer srv.Close()
client := newTestHTTPClient(t, srv.URL)
client.openAudio = func(string) (io.ReadCloser, error) {
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
}
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
if !errors.Is(err, sourceErr) {
t.Fatalf("Transcribe() error = %v, want source read failure", err)
}
}
func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
release := make(chan struct{})
source := newGatedReadCloser([]byte("audio-data"), release)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "bad request", http.StatusBadRequest)
}))
defer srv.Close()
client := newTestHTTPClient(t, srv.URL)
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
done := make(chan error, 1)
go func() {
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
done <- err
}()
select {
case err := <-done:
if err == nil {
t.Fatal("Transcribe() error = nil, want HTTP status error")
}
case <-time.After(time.Second):
t.Fatal("Transcribe() did not finish after server closed the request early")
}
}
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
release := make(chan struct{})
source := newGatedReadCloser([]byte("audio-data"), release)
firstByteReceived := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
part := firstMultipartFilePart(t, r)
buf := make([]byte, 1)
if _, err := part.Read(buf); err != nil {
t.Errorf("Read(file) error = %v", err)
return
}
close(firstByteReceived)
select {
case <-r.Context().Done():
case <-source.closed:
}
}))
defer srv.Close()
client := newTestHTTPClient(t, srv.URL)
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := client.Transcribe(ctx, TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
done <- err
}()
select {
case <-firstByteReceived:
cancel()
case <-time.After(time.Second):
cancel()
t.Fatal("server did not receive initial streamed audio")
}
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
}
case <-time.After(time.Second):
t.Fatal("Transcribe() did not finish after cancellation")
}
select {
case <-source.closed:
case <-time.After(time.Second):
t.Fatal("blocked audio source was not closed on cancellation")
}
}
func TestHTTPClientBoundsWhisperXResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
_, _ = w.Write([]byte(`{"ok":true}`))
}))
defer srv.Close()
client, err := NewHTTPClient(HTTPClientConfig{TranscribeURL: srv.URL, Language: "en", Timeout: time.Second, MaxResponseBytes: 4})
if err != nil {
t.Fatalf("NewHTTPClient() error = %v", err)
}
audioPath := writeWhisperXTestFile(t, "audio.flac", "audio-data")
_, err = client.Transcribe(context.Background(), TranscribeRequest{AudioPath: audioPath, OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
if err == nil || !strings.Contains(err.Error(), "whisperx response exceeds configured limit") {
t.Fatalf("Transcribe() error = %v, want bounded WhisperX response error", err)
}
}
func newTestHTTPClient(t *testing.T, endpoint string) *HTTPClient {
t.Helper()
client, err := NewHTTPClientFromConfigValues(endpoint, "en", "2s", "1ms", 0)
if err != nil {
t.Fatalf("NewHTTPClientFromConfigValues() error = %v", err)
}
return client
}
func firstMultipartFilePart(t *testing.T, r *http.Request) *multipart.Part {
t.Helper()
reader, err := r.MultipartReader()
if err != nil {
t.Fatalf("MultipartReader() error = %v", err)
}
part, err := reader.NextPart()
if err != nil {
t.Fatalf("NextPart() error = %v", err)
}
if part.FormName() != "file" {
t.Fatalf("first form field = %q, want file", part.FormName())
}
return part
}
type gatedReadCloser struct {
first []byte
release <-chan struct{}
closed chan struct{}
sent bool
once atomic.Bool
}
func newGatedReadCloser(first []byte, release <-chan struct{}) *gatedReadCloser {
return &gatedReadCloser{first: first, release: release, closed: make(chan struct{})}
}
func (r *gatedReadCloser) Read(p []byte) (int, error) {
if !r.sent {
r.sent = true
return copy(p, r.first), nil
}
select {
case <-r.release:
return 0, io.EOF
case <-r.closed:
return 0, errors.New("audio source closed")
}
}
func (r *gatedReadCloser) Close() error {
if r.once.CompareAndSwap(false, true) {
close(r.closed)
}
return nil
}
type failingReadCloser struct {
first []byte
err error
sent bool
}
func (r *failingReadCloser) Read(p []byte) (int, error) {
if !r.sent {
r.sent = true
return copy(p, r.first), nil
}
return 0, r.err
}
func (r *failingReadCloser) Close() error { return nil }
func writeWhisperXTestFile(t *testing.T, name, contents string) string {
t.Helper()

View File

@@ -5,6 +5,7 @@ import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
@@ -47,19 +48,55 @@ func (f *artifactSelectionFlag) Normalize() ([]string, error) {
}
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
if len(selected) == 0 {
return nil
_, err := resolveEffectiveArtifacts(cfg, selected)
return err
}
func resolveEffectiveArtifacts(cfg *config.Config, selected []string) (artifacts.EffectiveArtifactSet, error) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
if len(selected) == 0 {
return artifacts.ResolveEffectiveArtifactSet(nil, nil)
}
configured := cfg.Pipeline.Scriptorium.Artifacts
if len(configured) == 0 {
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
}
for _, name := range selected {
if _, ok := configured[name]; !ok {
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
configured := artifacts.ConfiguredArtifactDefinitions(cfg.Pipeline.Scriptorium.Artifacts)
if len(selected) > 0 && len(configured) == 0 {
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
}
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, selected)
if err != nil {
if strings.Contains(err.Error(), "is not configured") {
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts includes unknown artifact %q", selectedArtifactName(err))
}
return artifacts.EffectiveArtifactSet{}, err
}
if err := validateEffectiveArtifactConfiguration(cfg.Pipeline.Scriptorium.Artifacts, effective); err != nil {
return artifacts.EffectiveArtifactSet{}, err
}
return effective, nil
}
func selectedArtifactName(err error) string {
message := err.Error()
start := strings.Index(message, "\"")
if start < 0 {
return ""
}
end := strings.Index(message[start+1:], "\"")
if end < 0 {
return ""
}
return message[start+1 : start+1+end]
}
func validateEffectiveArtifactConfiguration(configured map[string]config.ScriptoriumArtifactConfig, effective artifacts.EffectiveArtifactSet) error {
for _, name := range effective.Keys() {
artifactCfg := configured[name]
if strings.TrimSpace(artifactCfg.PromptID) == "" {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when selected", name)
}
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when selected", name)
}
}
return nil

View File

@@ -21,7 +21,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
&stdout,
&stderr,
)
@@ -33,7 +33,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
}
}
func TestExecuteRunStageArchivePropagatesSelectedArtifacts(t *testing.T) {
func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
@@ -120,31 +120,32 @@ func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
}
}
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
store := &manifest.LocalStore{}
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
}
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
t.Fatalf("save manifest: %v", err)
}
var out bytes.Buffer
err := Resume(
err := Run(
context.Background(),
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
&out,
)
if err != nil {
t.Fatalf("Resume() error = %v", err)
t.Fatalf("Run() error = %v", err)
}
if !strings.Contains(out.String(), "has no remaining stages") {
t.Fatalf("output = %q, want no remaining stages", out.String())
if !strings.Contains(out.String(), "executed=1 skipped=11") {
t.Fatalf("output = %q, want all stages skipped", out.String())
}
}
@@ -281,7 +282,7 @@ func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
}
}
func TestExecutePublishForceRunsArchive(t *testing.T) {
func TestExecutePublishForceRunsPublish(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)

View File

@@ -110,6 +110,20 @@ func TestValidateSelectedArtifacts(t *testing.T) {
},
selected: []string{"player_handout", "session_recap"},
},
{
name: "selected disabled artifact must be executable",
cfg: &config.Config{
Pipeline: &config.PipelineConfig{
Scriptorium: &config.ScriptoriumConfig{
Artifacts: map[string]config.ScriptoriumArtifactConfig{
"player_handout": {Enabled: false, OutputPath: "artifacts/player_handout.md"},
},
},
},
},
selected: []string{"player_handout"},
wantErr: "pipeline.scriptorium.artifacts.player_handout.prompt_id is required when selected",
},
}
for _, tt := range tests {

View File

@@ -6,6 +6,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
@@ -33,12 +34,8 @@ func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFla
}
func validateCampaignIDToken(campaignID string) error {
if filepath.IsAbs(campaignID) ||
strings.Contains(campaignID, "/") ||
strings.Contains(campaignID, `\`) ||
campaignID == "." ||
campaignID == ".." {
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
if err := pathsafe.ValidateOpaqueSegment(campaignID); err != nil {
return fmt.Errorf("campaign id %q must be a single path segment and opaque identifier: %w", campaignID, err)
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More