Compare commits
453 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 70f1dbfb59 | |||
| f1575effc2 | |||
| 7e39a1d51f | |||
| 3722e572df | |||
| 7147e95252 | |||
| 9f2cfddce6 | |||
| 792cbd0677 | |||
| 5dca34e0e0 | |||
| 54d335c820 | |||
| 96b4e73f53 | |||
| 6e6054f744 | |||
| dc3288d15a | |||
| d872edca94 | |||
| 2ae63dbc2b | |||
| 26aa0cce29 | |||
| 7b0bf99ed1 | |||
| 89ada1b269 | |||
| 3438fe82c9 | |||
| a51a513804 | |||
| 995354f1a5 | |||
| 931c9591cd | |||
| 46096fc4ba | |||
| d3ecd907ee | |||
| 442469b9f5 | |||
| c45e042462 | |||
| 207da1b6ef | |||
| 9a1609435e | |||
| 13b83d7ee2 | |||
| a0312ed034 | |||
| 8ff74e1251 | |||
| 1b480d4ed5 | |||
| e13d87a077 | |||
| 5202f16972 | |||
| 5086596f94 | |||
| f7fc64db10 | |||
| 6b0778b420 | |||
| 5b92b3e790 | |||
| cc43906ae9 | |||
| dcaa2ad9c6 | |||
| 9d68cfcf2f | |||
| 2d346733d6 | |||
| cd75bf6d5a | |||
| e08de98ae6 | |||
| df213bc2ba | |||
| 869e68e834 | |||
| ed4623bc41 | |||
| 3ce198197d | |||
| 0ac8072a9a | |||
| 3ec54b8c72 | |||
| 47b3e5b55b | |||
| 99d1a0b926 | |||
| 884a62e8a6 | |||
| f7a40afc7a | |||
| a4f37ff427 | |||
| 2b97ef3722 | |||
| 63a86c43f4 | |||
| 92432232ef | |||
| d31ba5a410 | |||
| f0225e937c | |||
| 9e40c729e1 | |||
| 0e6ba414b3 | |||
| 10a2e89965 | |||
| f8e3e651e6 | |||
| 1558c9ccae | |||
| b68435128b | |||
| 5fb454a714 | |||
| 79cf9f1139 | |||
| 9bb203476d | |||
| 0726786016 | |||
| 204caf9d24 | |||
| 85663f5bc9 | |||
| f51b9bb046 | |||
| dd905c4daa | |||
| c3307ab5fb | |||
| 1484f51f48 | |||
| 51d21676c1 | |||
| c2806a1615 | |||
| 078f661e3a | |||
| adc60aa588 | |||
| 8d9eb5cc40 | |||
| 563acce012 | |||
| 49c359feb8 | |||
| 38202d4472 | |||
| 6d07dafc9f | |||
| 6c66e18c4b | |||
| 4048996652 | |||
| 470450e509 | |||
| 7e78c22fec | |||
| 610eaefb27 | |||
| 4bc33773fa | |||
| b74e31e1cc | |||
| 2f1e4407db | |||
| 4ccb4a73ca | |||
| 145baac393 | |||
| be59bff690 | |||
| 14b4fee214 | |||
| 7f84fc5d1e | |||
| a8b47f6185 | |||
| 5b62a8d60d | |||
| 0d4d71b9dc | |||
| 979d8049b1 | |||
| abd405015b | |||
| fddf78dc80 | |||
| 3136c26d4e | |||
| 4b84b4223d | |||
| 7a0e2cb11b | |||
| 74ab9218ff | |||
| da99857fd0 | |||
| a520cfdcc8 | |||
| 7febccf550 | |||
| 74977b4064 | |||
| 172e2d6a4f | |||
| 5c520dc0d4 | |||
| 6d18a3126e | |||
| 1e222a4db3 | |||
| 6a15052d77 | |||
| 2e69c14898 | |||
| dbda2a0161 | |||
| 2ee64f2eb1 | |||
| 8afe9427ca | |||
| e0f19833a1 | |||
| ca1b694aa9 | |||
| 9331373245 | |||
| ec072dc163 | |||
| f7ce1dcd56 | |||
| df275874e3 | |||
| 1da34ab5cd | |||
| c0fbc14f2d | |||
| 2d20900cbf | |||
| 7997f81f47 | |||
| 6d748d65d3 | |||
| 49324d3f35 | |||
| c98b4ef4ef | |||
| c8bae3a4a3 | |||
| ccbd7ee820 | |||
| e9c4873030 | |||
| 57719b7d81 | |||
| 62b1f9b0ae | |||
| 6c7a45d8d0 | |||
| 82d75e7beb | |||
| 61095e69ce | |||
| 2cfca710ec | |||
| 9679117e94 | |||
| d67a0389e0 | |||
| 9f84446509 | |||
| ec9b0e919d | |||
| 9bb7c1f4e3 | |||
| 3c092fbe34 | |||
| 3a7e2e5175 | |||
| 45b27ad562 | |||
| 1e418d6929 | |||
| 39b2665d8b | |||
| ba858cc997 | |||
| 0d57422b61 | |||
| b7d989f61f | |||
| 6813340f51 | |||
| eb788f17b8 | |||
| ebd3f415d3 | |||
| f15d8d8ce0 | |||
| 6c7bfd5e31 | |||
| 7c782a2603 | |||
| f1eef5b5e9 | |||
| f14583321d | |||
| 04283e99d4 | |||
| 7284228bec | |||
| da74d7ad18 | |||
| af5fbc1c4f | |||
| 5e4d2016e8 | |||
| 545aa6893b | |||
| 98139f7e8b | |||
| f8fa0a2623 | |||
| 9af773491b | |||
| 2a656f0f11 | |||
| aec35a2d9b | |||
| 23dc4e2078 | |||
| c812fe3655 | |||
| 3da97ca50c | |||
| 4c203d8588 | |||
| fcb5f825e1 | |||
| 4c57ace2f6 | |||
| dde7f76ecb | |||
| a102db36af | |||
| b5b1d22011 | |||
| c91599ef36 | |||
| 257f10c9fb | |||
| fb9a4d14f4 | |||
| c4435b76c4 | |||
| 61000a9466 | |||
| 7e4ceb3d48 | |||
| 4e991fa21d | |||
| f86b17045d | |||
| f302488075 | |||
| 8c1171478d | |||
| 7ee637803d | |||
| 4d6086fefb | |||
| 82cb53e107 | |||
| b97b12da7f | |||
| 49ea747b17 | |||
| 3b5e9db41f | |||
| 8b4b328c4e | |||
| ee2b8e63e6 | |||
| 51edd384c0 | |||
| b804d0f2c8 | |||
| fd5ccc668b | |||
| 0dc8ff9b52 | |||
| 8657a28bdb | |||
| a9c5e4ad4e | |||
| 2176b4371d | |||
| effc10d75b | |||
| 5887839aa1 | |||
| 3128bef20a | |||
| 4e4e2b7d96 | |||
| 99f4f9a0db | |||
| 6abdd67bb5 | |||
| c32e0c401f | |||
| ab5751459a | |||
| 62de6abdbf | |||
| 903dc70682 | |||
| 23c714da66 | |||
| 6639775d7d | |||
| 966b95b176 | |||
| 3bcf2c08dd | |||
| 700ab655ca | |||
| 85c5647385 | |||
| 2ef7c76d99 | |||
| 9bc1b0feda | |||
| 5cec84a4a7 | |||
| abfbe42d61 | |||
| e7e3bef1e4 | |||
| a68e8e31a4 | |||
| 3a9e60cda9 | |||
| 905ff03ccc | |||
| 495f7bcde4 | |||
| 51e0e8c5d0 | |||
| a2409a1fd1 | |||
| b3363f87d6 | |||
| 5831c0c9e6 | |||
| 42ed81cbe1 | |||
| e433c86203 | |||
| a2a144dffa | |||
| 8ef6e99d69 | |||
| 2545faef6c | |||
| 80be8be4d6 | |||
| 801adb385d | |||
| feba7b9d74 | |||
| b89224bbde | |||
| 131ffd9887 | |||
| af492c9e97 | |||
| f39fc94610 | |||
| 4e4eff6ba7 | |||
| 8ff1b4fa66 | |||
| 702f622e18 | |||
| 9da2c1e144 | |||
| 32653f54f9 | |||
| 72a200968a | |||
| b39b68add7 | |||
| d9fa1d9328 | |||
| 8375ad83f3 | |||
| 4158394dcf | |||
| eac7e155a5 | |||
| 0cf2cbfeb3 | |||
| 361dbb4ca8 | |||
| d6deccf3e8 | |||
| ee747243fe | |||
| a1ceb457e9 | |||
| 9900211fa4 | |||
| 60cebf0e4b | |||
| 7bd575187e | |||
| ab5a7e8e3d | |||
| 99b2e1cd81 | |||
| 363313d99c | |||
| 18ddf00d3d | |||
| 59f3fe3d1d | |||
| 1dccf5f140 | |||
| 0b40cf8026 | |||
| a7ec195587 | |||
| 13de820931 | |||
| 14ef59aaed | |||
| f387222fce | |||
| 9cb9008dfc | |||
| 083decc5b4 | |||
| 57cac5d3f7 | |||
| 0a772e03b4 | |||
| 0920062a38 | |||
| 39afe644eb | |||
| e3ee3de10a | |||
| 9c72db56e9 | |||
| bb2d606dbb | |||
| 9850767a8a | |||
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c | |||
| df58595d1e | |||
| c3c14e7468 | |||
| e7319ea016 | |||
| bd2d5e2496 | |||
| 115a44f629 | |||
| 18411dc5b5 | |||
| e23dc1ab6e | |||
| e1359ea227 | |||
| 7fdd99ec27 | |||
| a90231ce0c | |||
| ed879b8bb0 | |||
| 717451512a | |||
| 3ddb3a947b | |||
| c6632d5576 | |||
| ffc07922c7 | |||
| f3310d4d16 | |||
| 88cee96d8d | |||
| 2fece10215 | |||
| 0658f2f642 | |||
| a51228c803 | |||
| 4491fb5ccd | |||
| 30b905765c | |||
| 03eac70881 | |||
| 0f7e6b979f | |||
| c366912586 | |||
| 9fe44cd00d | |||
| 094b0d2532 | |||
| 98649f4d81 | |||
| 8a559efd5b | |||
| 72deccb4e2 | |||
| 5620fc5bcf | |||
| be57e675e0 | |||
| 3971443831 | |||
| a6b0c33e9f | |||
| 96b886e711 | |||
| 7d584ee6cd | |||
| 572a112c31 | |||
| ea87c335d6 | |||
| 7169ff04df | |||
| ef1f650bc0 | |||
| 0d02cb9fa0 | |||
| 0299b128cf | |||
| d723384888 | |||
| 54228055c8 | |||
| 23ed716450 | |||
| ab59bab044 | |||
| 71395bb076 | |||
| 79737edf79 | |||
| df2c765b7f | |||
| f050b9dd54 | |||
| 9c9cb54339 | |||
| 7657ec3ad6 | |||
| cee52aa092 | |||
| e920f3a8d5 | |||
| 591c529a09 | |||
| 7324c5a686 | |||
| d0936fb022 | |||
| 2aa074c5cf | |||
| 782d0cf3b9 | |||
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 | |||
| bc2ade38d9 | |||
| 5be831eb13 | |||
| cae4d99a89 | |||
| e09dc0512d | |||
| ae82bc1ce0 | |||
| 01eb7aa1aa | |||
| 2ca700195c | |||
| 2b08c34539 | |||
| 79f1fc1e09 | |||
| 9c753270bd | |||
| b907cb01aa | |||
| 7824afd4a5 | |||
| 2a4e1e912c | |||
| dd03c09d75 | |||
| 5bc8e8683f | |||
| 648001a8fe | |||
| 6684774f52 | |||
| f3b63bd5e5 | |||
| 23d6470b0f | |||
| 128449040f | |||
| 02ab106ade | |||
| c128970f58 | |||
| d001baa660 | |||
| c5c35cd3b4 | |||
| 574b1cde6c | |||
| ebb21b9201 | |||
| 958f446387 | |||
| 86caf4b222 | |||
| e38ed8ba97 | |||
| 3e79cf4724 | |||
| 859ae1ae10 | |||
| c63ecbab32 | |||
| 8480b74283 | |||
| 087869f7fa | |||
| 2b2a314d65 | |||
| 08b0f4edc5 | |||
| 571a289296 | |||
| 9f80635b42 | |||
| 11a3e174b6 | |||
| 9c5e5d6dc1 | |||
| c4e87f58c7 | |||
| 37daab7857 | |||
| 2356688cb9 | |||
| 1054b64d9f | |||
| 01fb02426c | |||
| 7dc79e052f | |||
| cb525c0f72 | |||
| 622677d038 | |||
| 550288e008 | |||
| e58e545686 | |||
| 6ff54c5a0f | |||
| 924b5d15c6 | |||
| b065663180 | |||
| 3ba564b00f | |||
| a3986cf0d6 | |||
| 539601bd16 | |||
| 6ca1c8d6b0 | |||
| 4b7b50981b | |||
| 33f7ae8f2e | |||
| d5a9ad38f8 | |||
| 6fbefb9867 | |||
| 1665359486 | |||
| 03f2543927 | |||
| fe9c348092 | |||
| f7f8f1a949 | |||
| d40c91acde | |||
| ed4dcf1ef7 | |||
| 24cce49a70 | |||
| 1e6db89dd4 | |||
| 0454296c81 | |||
| 58c6ab2d54 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -2,6 +2,8 @@
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
.DS_Store
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
@@ -22,6 +24,9 @@ AGENTS.md
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go cache
|
||||
.gocache
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
@@ -2,39 +2,26 @@ when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
validate-release:
|
||||
image: golang:1.26.7
|
||||
commands:
|
||||
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
|
||||
|
||||
build-release-assets:
|
||||
image: golang:1.26.7
|
||||
depends_on:
|
||||
- validate-release
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
case "$PWD" in
|
||||
/*) ;;
|
||||
*) echo "release workspace must have an absolute path" >&2; exit 1 ;;
|
||||
esac
|
||||
./scripts/build-release-assets.sh "$CI_COMMIT_TAG" "$PWD/dist"
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/narratio/cmd/narratio"
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
goarch="$2"
|
||||
suffix="$3"
|
||||
output="$dist/narratio-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/narratio/internal/buildinfo.Version=$version" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
publish-release:
|
||||
image: woodpeckerci/plugin-release:0.3.1
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
settings:
|
||||
@@ -42,6 +29,8 @@ steps:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/narratio-*
|
||||
title: Narratio ${CI_COMMIT_TAG}
|
||||
note: docs/releases/${CI_COMMIT_TAG}.md
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
|
||||
8
.woodpecker/shuffle.yml
Normal file
8
.woodpecker/shuffle.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
when:
|
||||
- event: cron
|
||||
|
||||
steps:
|
||||
shuffled-race-tests:
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test -race -shuffle=on -count=3 ./...
|
||||
47
.woodpecker/verify.yml
Normal file
47
.woodpecker/verify.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
when:
|
||||
- event: [push, pull_request]
|
||||
|
||||
steps:
|
||||
tests:
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test ./...
|
||||
|
||||
race-tests:
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test -race ./...
|
||||
|
||||
static-analysis:
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go vet ./...
|
||||
|
||||
build:
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go build ./...
|
||||
|
||||
documentation-and-examples:
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test ./internal/doccheck
|
||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
|
||||
cross-build:
|
||||
image: golang:1.26
|
||||
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
output_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$output_dir"' EXIT
|
||||
for target in linux/amd64 linux/arm64 darwin/arm64; do
|
||||
goos="${target%/*}"
|
||||
goarch="${target#*/}"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||
done
|
||||
322
README.md
322
README.md
@@ -1,300 +1,42 @@
|
||||
# narratio
|
||||
|
||||
`narratio` is a Go orchestration application for processing D&D session audio into transcripts and generated artifacts.
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts, validated Notarius extraction lanes, and generated
|
||||
artifacts.
|
||||
|
||||
## Current Implementation
|
||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||
publish, and restore support.
|
||||
|
||||
Implemented now:
|
||||
|
||||
- strict config loading/validation (`pipeline.yml` and `session.yml`)
|
||||
- local workspace/session layout, locking, and manifest persistence
|
||||
- resumable stage control (`run`, `plan`, `resume`, `run-stage`, `status`)
|
||||
- real `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` stages
|
||||
- real WhisperX, Seriatim, and Audita adapters
|
||||
- real Scriptorium subprocess adapter
|
||||
- optional Scriptorium render diagnostics (`render_debug`)
|
||||
|
||||
Not implemented yet:
|
||||
|
||||
- `archive` stage behavior
|
||||
- `notify` stage behavior
|
||||
- additional analyze artifacts beyond `session_recap`
|
||||
- generic DAG orchestration
|
||||
|
||||
## Config Files
|
||||
|
||||
Narratio expects two YAML files:
|
||||
|
||||
- `pipeline.yml`: pipeline/workspace settings
|
||||
- `session.yml`: per-session settings
|
||||
|
||||
Pipeline config lookup for CLI commands:
|
||||
|
||||
- if `--config <path>` is provided, Narratio uses that path
|
||||
- if `--config` is omitted, Narratio searches in this order:
|
||||
- `/usr/local/etc/narratio/pipeline.yml`
|
||||
- `/etc/narratio/pipeline.yml`
|
||||
|
||||
Optional secrets-from-files config:
|
||||
|
||||
- `pipeline.secrets.env_dir` may point to a directory of secret files
|
||||
- each top-level file with an env-var-style name is loaded as an environment variable:
|
||||
- file name = env var name
|
||||
- file contents = env var value (trailing newline/CRLF trimmed)
|
||||
- process environment wins: existing env vars are not overwritten
|
||||
- if configured, Narratio fails fast when `env_dir` is missing/unreadable
|
||||
- relative `env_dir` values resolve from Narratio’s current working directory
|
||||
|
||||
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
||||
|
||||
## Canonical Stage Order
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
## Transcript Tiers
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output
|
||||
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
## Normalize Configuration
|
||||
|
||||
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
Allowed `normalize.output_schema` values:
|
||||
|
||||
- `seriatim-minimal`
|
||||
- `seriatim-intermediate`
|
||||
- `seriatim-full`
|
||||
|
||||
`normalize.output_path` is treated as session-workdir-relative when not absolute.
|
||||
|
||||
Normalize stage behavior summary:
|
||||
|
||||
- normalize runs after `polish` and before `trim`
|
||||
- normalize resolves `transcripts/processed.json`
|
||||
- normalize runs Seriatim `normalize` to produce `transcripts/normalized.json`
|
||||
- normalize diagnostics are written to:
|
||||
- `artifacts/seriatim.normalize.report.json` (when enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## Trim Configuration
|
||||
|
||||
`pipeline.trim` is optional. If omitted, no trim config is loaded. If `trim.enabled` is omitted, it defaults to `false`.
|
||||
|
||||
When `trim.enabled: true`:
|
||||
|
||||
- `trim.output_path` is required
|
||||
- `trim.bounds.prompt_id` is required
|
||||
- `trim.bounds.transcript_input_name` is required
|
||||
- `trim.bounds.output_path` is required
|
||||
- `trim.bounds.timeout` must be a valid Go duration when provided
|
||||
- `trim.bounds.render_debug: true` requires `trim.bounds.render_output_path`
|
||||
- `trim.bounds.profile_id` may be empty to use the prompt default profile
|
||||
|
||||
Trim paths are treated as session-workdir-relative when not absolute.
|
||||
|
||||
Example trim config:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```sh
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Trim behavior summary:
|
||||
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete
|
||||
`session.yml` files or their explicit command-line alternatives.
|
||||
|
||||
- trim discovers and validates `transcripts/normalized.json`
|
||||
- trim uses Scriptorium bounds (`dnd_session.bounds` by example config) to produce `artifacts/session_bounds.json`
|
||||
- bounds IDs are validated against the same normalized transcript ID space that Seriatim trim will consume
|
||||
- trim converts bounds to Seriatim keep selector (for example `10-868`) and runs Seriatim trim
|
||||
- if trim is disabled, Narratio copies normalized transcript to trimmed transcript and records `trim_action=copy_disabled`
|
||||
## Documentation
|
||||
|
||||
Trim outputs and diagnostics:
|
||||
- [CLI reference](docs/cli.md) — commands, arguments, flags, and invocation
|
||||
behavior.
|
||||
- [Configuration](docs/config.md) — discovery, fields, defaults, and
|
||||
validation.
|
||||
- [Operations](docs/operations.md) — runtime workflow, state, publishing,
|
||||
recovery, and cleanup.
|
||||
- [Troubleshooting](docs/troubleshooting.md) — symptom-driven diagnosis and
|
||||
safe remedies.
|
||||
- [Integration contracts](docs/integrations/) — external tools, formats, and
|
||||
compatibility expectations.
|
||||
- [Maintained examples](examples/README.md) — complete copyable configuration
|
||||
and input files, including the production/testing split bundle.
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
## Maintainer Documentation
|
||||
|
||||
Render-debug files are diagnostics and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## Scriptorium Configuration
|
||||
|
||||
`pipeline.scriptorium` is optional. When present, Narratio validates and uses it for analyze-stage artifact generation.
|
||||
|
||||
Key points:
|
||||
|
||||
- `scriptorium.binary` is required when section is present
|
||||
- `scriptorium.config_path` is optional
|
||||
- `scriptorium.timeout` defaults to `10m` when omitted
|
||||
- `scriptorium.render_debug` enables render diagnostics globally
|
||||
- artifacts are configured under `scriptorium.artifacts` (map shape supports multiple artifacts)
|
||||
- enabled artifacts require `prompt_id` and `output_path`
|
||||
- artifact `render_debug` may override global render setting
|
||||
- `vars` currently support boolean and string values
|
||||
|
||||
Example `session_recap` artifact definition:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: "scriptorium"
|
||||
config_path: "/etc/scriptorium/config.yml"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
profile_id: "local-quality" # optional
|
||||
output_path: "artifacts/session_recap.md"
|
||||
timeout: "10m"
|
||||
# render_debug: true # optional per-artifact override
|
||||
|
||||
inputs:
|
||||
transcript:
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
path: "" # optional; set when available
|
||||
required: false
|
||||
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
```
|
||||
|
||||
Prompt IDs and profile IDs are configuration values. They are not hardcoded in analyze-stage logic.
|
||||
|
||||
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
||||
|
||||
If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs.
|
||||
|
||||
## Scriptorium Runtime Behavior
|
||||
|
||||
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
||||
|
||||
- generation: `scriptorium run`
|
||||
- diagnostics/testing: `scriptorium render --format json` when `render_debug` is enabled
|
||||
|
||||
For the initial implementation, only `session_recap` generation is supported.
|
||||
|
||||
Analyze-stage session recap behavior:
|
||||
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
||||
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
|
||||
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
|
||||
- missing trimmed transcript fails clearly and advises running trim stage first
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains supported for advanced/debug use cases
|
||||
- optionally includes `previous_recap` when configured and resolvable
|
||||
- omits optional previous recap when unavailable
|
||||
- fails if required inputs are missing
|
||||
- validates output file exists and is non-empty
|
||||
|
||||
Expected session output paths:
|
||||
|
||||
- `artifacts/session_recap.md`
|
||||
- `logs/scriptorium.session_recap.stdout.log`
|
||||
- `logs/scriptorium.session_recap.stderr.log`
|
||||
- `config/scriptorium.session_recap.generated.yml`
|
||||
- `artifacts/session_recap.render.json` when render diagnostics are enabled
|
||||
|
||||
## Examples
|
||||
|
||||
Starter files:
|
||||
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/session.minimal.yml`
|
||||
- `examples/speakers.yml`
|
||||
|
||||
## Commands
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Plan a run:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio plan --session examples/session.minimal.yml
|
||||
```
|
||||
|
||||
Use `--config <path>` to override default pipeline lookup when needed.
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
||||
```
|
||||
|
||||
Run analyze only:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
||||
```
|
||||
|
||||
## Operational Note
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime config change, rerun the appropriate upstream stages before relying on downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary/autocorrect/speaker-context changes: rerun at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim bounds prompt/profile/config changes: rerun at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes: rerun `analyze`
|
||||
|
||||
## Roadmap
|
||||
|
||||
Near-term roadmap:
|
||||
|
||||
- extend analyze to additional configured artifacts
|
||||
- support workflows where later artifacts consume earlier generated artifacts
|
||||
- keep orchestration explicit without a generic DAG engine
|
||||
- implement archive and notify backends
|
||||
- [Development guide](docs/development.md) — first-read orientation and
|
||||
task-specific reading routes.
|
||||
- [Internal overview](docs/internal/overview.md) — implemented component map.
|
||||
- [Architecture](docs/policy/architecture.md) — normative boundaries and
|
||||
invariants.
|
||||
- [Documentation policy](docs/policy/documentation.md) — canonical ownership
|
||||
and maintenance rules.
|
||||
- [Testing policy](docs/policy/testing.md) — test value, boundaries, and
|
||||
sufficiency.
|
||||
|
||||
333
architecture.md
333
architecture.md
@@ -1,333 +0,0 @@
|
||||
# Narratio Architecture
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
`narratio` is a Go orchestrator for D&D session processing. It runs a stage-based local pipeline from audio input through transcript processing and artifact generation, with manifest-based skip/force/resume behavior.
|
||||
|
||||
Narratio integrates with Scriptorium through the **public CLI** (`scriptorium run` and `scriptorium render`) via synchronous subprocess execution.
|
||||
|
||||
## 2. Current Status
|
||||
|
||||
Implemented:
|
||||
|
||||
- strict `pipeline.yml` + `session.yml` loading with strict YAML field checking (`KnownFields(true)`)
|
||||
- local workspace/session layout, lock file handling, artifact path helpers, checksums, and atomic writes
|
||||
- manifest store and stage status transitions for resumable runs
|
||||
- real `prepare`, `transcribe`, `merge`, and `polish` stages
|
||||
- real WhisperX HTTP adapter
|
||||
- real Seriatim subprocess adapter
|
||||
- real Audita subprocess adapter
|
||||
- real Scriptorium subprocess adapter
|
||||
- real `normalize` stage producing `transcripts/normalized.json`
|
||||
- real `trim` stage producing `transcripts/trimmed.json`
|
||||
- real `analyze` stage for initial `session_recap` generation
|
||||
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
||||
|
||||
Still placeholder/future:
|
||||
|
||||
- `archive` stage behavior
|
||||
- `notify` stage behavior
|
||||
- additional Scriptorium artifact types beyond `session_recap`
|
||||
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
||||
- generic stale detection based on input/config checksums
|
||||
|
||||
## 3. Pipeline and Stage Boundaries
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
Boundary rules:
|
||||
|
||||
- orchestration logic lives in `internal/app`
|
||||
- stage business logic lives in `internal/stage`
|
||||
- external-tool CLI construction lives in adapter packages
|
||||
- Scriptorium CLI details stay in `internal/adapters/scriptorium`
|
||||
|
||||
## 4. Scriptorium Integration Model
|
||||
|
||||
Integration mode:
|
||||
|
||||
- public CLI subprocesses only (no Scriptorium internal Go packages, no HTTP API)
|
||||
- production generation uses `scriptorium run`
|
||||
- diagnostics/testing render uses `scriptorium render --format json`
|
||||
|
||||
Run invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium run --prompt <prompt_id> --input name=path --out <output_path>
|
||||
```
|
||||
|
||||
Optional flags passed when configured:
|
||||
|
||||
- `--config <path>`
|
||||
- `--profile <profile_id>`
|
||||
- repeated `--var name=value`
|
||||
- repeated `--input name=path`
|
||||
- `--timeout <duration>`
|
||||
- `--api-key-env <ENV_NAME>` when configured
|
||||
|
||||
Render invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium render --prompt <prompt_id> --input name=path --format json --out <render_output_path>
|
||||
```
|
||||
|
||||
Adapter behavior:
|
||||
|
||||
- always passes `--out`
|
||||
- captures stdout/stderr separately
|
||||
- writes generated invocation metadata YAML (redacted, no secrets)
|
||||
- treats exit code `0` as success
|
||||
- treats exit code `1` as failure
|
||||
- treats exit code `2` as failure with `validation_failed=true` and preserves output metadata when available
|
||||
- validates successful output files exist and are non-empty
|
||||
- does not treat non-empty stderr as failure by itself
|
||||
|
||||
## 5. Configuration Contract
|
||||
|
||||
CLI pipeline config path resolution:
|
||||
|
||||
- when `--config <path>` is provided, that path is used
|
||||
- when `--config` is omitted, Narratio searches defaults in order:
|
||||
- `/usr/local/etc/narratio/pipeline.yml`
|
||||
- `/etc/narratio/pipeline.yml`
|
||||
|
||||
Optional pipeline secrets directory:
|
||||
|
||||
- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution
|
||||
- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed)
|
||||
- only env-var-style file names are considered; other entries are ignored
|
||||
- existing process environment values are preserved (not overwritten)
|
||||
- if configured, unreadable/missing `env_dir` fails command execution early
|
||||
- relative `env_dir` values are resolved from current working directory
|
||||
|
||||
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
||||
|
||||
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
||||
|
||||
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
||||
|
||||
When `pipeline.normalize` is omitted, defaults are applied:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
When `pipeline.normalize` is present:
|
||||
|
||||
- `output_path` must be non-empty
|
||||
- `output_schema` must be one of `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`
|
||||
- relative `output_path` values are session-workdir-relative paths
|
||||
- Seriatim binary settings still come from `pipeline.seriatim`
|
||||
|
||||
When `pipeline.trim` is present:
|
||||
|
||||
- `enabled` is optional and defaults to `false` when omitted
|
||||
- relative `output_path`, `bounds.output_path`, and `bounds.render_output_path` values are session-workdir-relative paths
|
||||
- do not store secrets in trim config values
|
||||
|
||||
When `pipeline.trim.enabled: true`:
|
||||
|
||||
- `output_path` is required and non-empty
|
||||
- `bounds.prompt_id` is required and non-empty
|
||||
- `bounds.transcript_input_name` is required and non-empty
|
||||
- `bounds.output_path` is required and non-empty
|
||||
- `bounds.timeout` must parse as a Go duration when provided
|
||||
- `bounds.render_debug: true` requires non-empty `bounds.render_output_path`
|
||||
- `bounds.profile_id` may be empty to use the prompt default profile
|
||||
- prompt IDs are config values, not hardcoded stage logic
|
||||
|
||||
When `pipeline.scriptorium` is present:
|
||||
|
||||
- `binary` is required and non-empty
|
||||
- `config_path` is optional; when provided it must be non-empty
|
||||
- `timeout` is optional; when provided it must parse as a Go duration
|
||||
- default `timeout` is `10m`
|
||||
- unknown YAML fields fail strict decode
|
||||
|
||||
Artifacts are configured as a map under `pipeline.scriptorium.artifacts` so multiple artifacts are possible in the config shape.
|
||||
|
||||
For each artifact definition:
|
||||
|
||||
- `enabled: true` requires non-empty `prompt_id`
|
||||
- `enabled: true` requires non-empty `output_path`
|
||||
- `timeout` must parse as Go duration when present
|
||||
- optional per-artifact `render_debug` may override global `scriptorium.render_debug`
|
||||
- `inputs` are named and each input requires non-empty `source`
|
||||
- inputs may be optional (`required: false`)
|
||||
- `vars` values currently support `string` and `bool`
|
||||
|
||||
Prompt IDs and profile IDs are configuration values, not hardcoded stage logic.
|
||||
|
||||
Trim config shape:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```
|
||||
|
||||
## 6. Transcript Tiers
|
||||
|
||||
Narratio currently produces and uses four transcript tiers:
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output (includes pre/post-game content)
|
||||
- `transcripts/normalized.json`: normalized transcript generated by Seriatim normalize
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
Trim reads `transcripts/normalized.json`, validates bounds IDs against that same transcript ID space, and writes `transcripts/trimmed.json`.
|
||||
|
||||
## 7. Normalize Stage (Current Implementation)
|
||||
|
||||
Normalize stage behavior:
|
||||
|
||||
- stage order position: after `polish` and before `trim`
|
||||
- discovers processed transcript from manifest polish outputs (`transcript_processed`) when present, else `work/<session_id>/transcripts/processed.json`
|
||||
- validates processed transcript JSON shape (`segments` array required)
|
||||
- runs Seriatim `normalize` to produce normalized transcript
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- validates normalize report JSON when enabled
|
||||
|
||||
Expected normalize outputs and diagnostics:
|
||||
|
||||
- `transcripts/normalized.json`
|
||||
- `artifacts/seriatim.normalize.report.json` (when normalize report is enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## 8. Trim Stage (Current Implementation)
|
||||
|
||||
Trim stage behavior:
|
||||
|
||||
- stage order position: after `normalize` and before `analyze`
|
||||
- discovers normalized transcript from manifest normalize outputs (`transcript_normalized`) when present, else `work/<session_id>/transcripts/normalized.json`
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- when `trim.enabled: false` (or trim config omitted), deterministically copies normalized transcript to `transcripts/trimmed.json` and records `trim_action=copy_disabled`
|
||||
- when `trim.enabled: true`:
|
||||
- runs Scriptorium bounds prompt using configured `trim.bounds.prompt_id`
|
||||
- writes bounds output to configured path (typically `artifacts/session_bounds.json`)
|
||||
- parses and validates bounds output against the same normalized transcript being trimmed
|
||||
- converts bounds range to Seriatim keep selector (for example `10-868`)
|
||||
- runs Seriatim `trim` to produce `transcripts/trimmed.json`
|
||||
- supports no-trim bounds actions (`none`/`copy`) by copying normalized transcript unchanged
|
||||
- validates trimmed transcript JSON shape (`segments` array required)
|
||||
|
||||
Expected trim outputs and diagnostics:
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs when enabled:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
|
||||
Render-debug files are diagnostics. They are recorded in stage metadata/log/config refs and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## 9. Analyze Stage (Current Implementation)
|
||||
|
||||
The current real analyze implementation supports only `scriptorium.artifacts.session_recap`.
|
||||
|
||||
Behavior:
|
||||
|
||||
- if `pipeline.scriptorium` is missing, analyze returns a skipped result with metadata
|
||||
- if no Scriptorium artifacts are enabled, analyze returns a skipped result with metadata
|
||||
- if enabled artifacts exist but `session_recap` is not enabled, analyze fails clearly
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- `session_recap` should use `trimmed_transcript` input (`transcripts/trimmed.json`) for in-universe recap generation
|
||||
- `trimmed_transcript` input is resolved from manifest (`trim` output kind `transcript_trimmed`) when available, otherwise fallback path `work/<session_id>/transcripts/trimmed.json`
|
||||
- `normalized_transcript` input is resolved from manifest (`normalize` output kind `transcript_normalized`) when available, otherwise fallback path `work/<session_id>/transcripts/normalized.json`
|
||||
- `processed_transcript` input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work/<session_id>/transcripts/processed.json`
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains available for advanced/debug use cases
|
||||
- transcript inputs are validated as JSON with top-level `segments` array
|
||||
- configured inputs are resolved by source
|
||||
- optional `previous_recap` is omitted when unavailable
|
||||
- required `previous_recap` fails before invocation when unavailable
|
||||
- vars are built from config + session metadata
|
||||
- `render_debug` controls pre-run `scriptorium render` diagnostics
|
||||
- render failure stops stage before production run
|
||||
- render output is validated as JSON
|
||||
- production call uses Scriptorium adapter `RunArtifact`
|
||||
- successful run output must exist and be non-empty
|
||||
- missing `trimmed_transcript` input for configured `trimmed_transcript` source fails clearly with guidance to run trim stage first
|
||||
- manifest records output refs, logs, generated config paths, and non-secret provenance metadata
|
||||
|
||||
## 10. Session Recap Paths
|
||||
|
||||
Current expected paths for `session_recap`:
|
||||
|
||||
- artifact output: `artifacts/session_recap.md`
|
||||
- run stdout log: `logs/scriptorium.session_recap.stdout.log`
|
||||
- run stderr log: `logs/scriptorium.session_recap.stderr.log`
|
||||
- run generated invocation/config: `config/scriptorium.session_recap.generated.yml`
|
||||
- render output (when enabled): `artifacts/session_recap.render.json`
|
||||
- render stdout log: `logs/scriptorium.session_recap.render.stdout.log`
|
||||
- render stderr log: `logs/scriptorium.session_recap.render.stderr.log`
|
||||
- render generated invocation/config: `config/scriptorium.session_recap.render.generated.yml`
|
||||
|
||||
## 11. Security and Privacy
|
||||
|
||||
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
||||
- if API-key integration is configured, pass env var names only (never raw key values)
|
||||
- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs
|
||||
- avoid logging transcript content or rendered prompt content by default
|
||||
- treat generated artifacts and logs as potentially sensitive session material
|
||||
|
||||
## 12. Operational Caveat (Pre-Stale-Detection)
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime configuration change (for example glossary files, prompt IDs, profile IDs, or relevant pipeline settings), rerun the appropriate prior stages to refresh downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary or autocorrect changes usually require rerunning at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim prompt/profile changes require rerunning at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes require rerunning `analyze`
|
||||
|
||||
## 13. Roadmap
|
||||
|
||||
Planned next steps:
|
||||
|
||||
- extend analyze beyond `session_recap` to additional configured artifacts
|
||||
- support artifact inputs that consume prior generated artifacts
|
||||
- keep this composable without adding a generic DAG engine in the near term
|
||||
- implement real `archive` backend behavior
|
||||
- implement real `notify` backend behavior
|
||||
- add checksum-based stale detection and stale transitions
|
||||
|
||||
Architectural invariants remain:
|
||||
|
||||
- strict config decoding/validation
|
||||
- manifest-driven run control
|
||||
- clear stage/adapter separation
|
||||
- configuration-driven prompt/profile/input/vars/output mapping
|
||||
- Scriptorium integration through public CLI subprocess contract
|
||||
485
docs/cli.md
Normal file
485
docs/cli.md
Normal file
@@ -0,0 +1,485 @@
|
||||
# CLI Reference
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
This runs the canonical full pipeline for session `2026-04-04`.
|
||||
|
||||
## Command Overview
|
||||
|
||||
Top-level commands:
|
||||
|
||||
- `version`: print the Narratio build version.
|
||||
- `run <session_id>`: run all or one contiguous range of the canonical stage order.
|
||||
- `regenerate-artifacts <session_id>`: force-run extraction through analysis.
|
||||
- `run-stage <stage> <session_id>`: run one stage.
|
||||
- `analyze <session_id>`: force-run analyze.
|
||||
- `publish <session_id>`: force-run publish.
|
||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||
- `session <subcommand>`: session helper commands.
|
||||
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
|
||||
|
||||
Session subcommands:
|
||||
|
||||
- `session init <session_id>`
|
||||
- `session plan <session_id>`
|
||||
- `session validate <session_id>`
|
||||
- `session status <session_id>`
|
||||
- `session restore <session_id>`
|
||||
- `session artifacts <session_id>`
|
||||
- `session locks <session_id>`
|
||||
- `session locks add <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>`
|
||||
|
||||
## Common Config Flags
|
||||
|
||||
Most session-aware commands accept:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>`
|
||||
- `--campaign-file <campaign.yml>`
|
||||
- `--session <session.yml>`
|
||||
- `--session-id <session_id>`
|
||||
- `--previous-session-id <session_id>`
|
||||
- `--profile <name>`
|
||||
- `--secrets-dir <directory>`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||
- `--session` is not used by `session init`.
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `--previous-session-id` is a strict expectation: the selected session file
|
||||
must contain the same `previous_session_id`.
|
||||
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
|
||||
may be supplied once; omitted, empty, and the exact name `default` select the
|
||||
base configuration, while an unknown named profile fails. The same selection
|
||||
applies to all common-flag commands, including `regenerate-artifacts`.
|
||||
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
|
||||
relative value uses caller CWD. Without the flag, a present `secrets/`
|
||||
directory beside the resolved root pipeline is used; an absent one is skipped.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
- notification delivery is currently limited to the configured `noop` mode; see
|
||||
the [configuration reference](./config.md#notifications).
|
||||
|
||||
## Session ID Input Rules
|
||||
|
||||
Session-aware commands accept one of these forms:
|
||||
|
||||
- positional session ID: `... <session_id>`
|
||||
- compatibility flag: `... --session-id <session_id>`
|
||||
|
||||
When both are present, command parsing requires an exact match.
|
||||
|
||||
Commands with additional positionals keep their command-specific order:
|
||||
|
||||
- `run-stage <stage> <session_id>` or `run-stage <stage> --session-id <session_id>`
|
||||
- `session locks add <session_id> <source>` or `session locks add --session-id <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>` or `session locks remove --session-id <session_id> <source>`
|
||||
|
||||
## Command Reference
|
||||
|
||||
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
|
||||
|
||||
```bash
|
||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
|
||||
```
|
||||
|
||||
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
||||
a campaign is selected—the campaign-owned party. They neither discover or load
|
||||
a session nor create a workspace, manifest, run, lock, adapter, or remote
|
||||
connection. Like every pipeline-loading command, they bootstrap the selected or
|
||||
conventional secrets directory before resolving configuration; `config show`
|
||||
never includes bootstrap reports or secret values in its YAML output.
|
||||
Typed credential references, including `notarius.llm_api_key`, must therefore
|
||||
resolve even though these inspection commands never construct an adapter.
|
||||
|
||||
Campaign selection is optional for a pipeline without party-driven artifact
|
||||
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
||||
configured default campaign so Narratio can expand its concrete artifacts and
|
||||
publish rules. `--campaign` and `--campaign-file` remain mutually exclusive.
|
||||
Session, range, force, and artifact-execution flags are not accepted.
|
||||
|
||||
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
||||
effective-digest summary after successful complete validation. `config show`
|
||||
writes one deterministic, secret-free YAML document containing defaulted and
|
||||
expanded concrete configuration. It omits ConfigKit directives, artifact
|
||||
family declarations, and runtime provenance.
|
||||
|
||||
`config sources` reports the same fully validated resolution without printing
|
||||
effective values. Its header identifies the root, ordered imports, selected
|
||||
profile and overlay, selected campaign, party source, and digest. The
|
||||
remaining tab-separated records are sorted as `path`, `role`, and `source`.
|
||||
Roles distinguish root, import, profile, centralized default, campaign, party,
|
||||
and generated family ownership. A generated party member has
|
||||
one family record and one party record at the same logical path. The output
|
||||
never reads or prints secret values.
|
||||
|
||||
`config artifacts` prints the concrete catalog used by analysis, ordered by
|
||||
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
||||
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||
lists, and control characters in fields are escaped. Family-member rows identify
|
||||
their family, character, defining family file, and party file. An empty catalog
|
||||
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
||||
party-driven families retain the campaign requirement. See
|
||||
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||
configuration layout.
|
||||
|
||||
`config diff` resolves both supplied profile names independently and compares
|
||||
their fully resolved, secret-free effective mappings. It does
|
||||
not accept `--profile`; the two positional names must be distinct, declared
|
||||
profiles. When party-driven families are present, both profiles must resolve to
|
||||
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
||||
campaign configuration would otherwise select different files.
|
||||
|
||||
Equal profiles print `no differences`. Otherwise, sorted tab-separated records
|
||||
use one of these forms, with compact JSON values:
|
||||
|
||||
```text
|
||||
added <path> <right-value>
|
||||
removed <path> <left-value>
|
||||
changed <path> <left-value> <right-value>
|
||||
```
|
||||
|
||||
Mappings are flattened to their logical field paths; lists remain one atomic
|
||||
value. The command compares defaulted concrete artifacts and publish rules, not
|
||||
profile names, source-file layout, or formatting. It succeeds when differences
|
||||
are found, making it suitable for review and migration checks.
|
||||
|
||||
### `version`
|
||||
|
||||
```bash
|
||||
narratio version
|
||||
```
|
||||
|
||||
Official release binaries report their exact Git tag. Binaries built directly
|
||||
from source without release linker metadata report `dev`.
|
||||
|
||||
### `run`
|
||||
|
||||
```bash
|
||||
narratio run <session_id> [--from <stage>] [--through <stage>] [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- evaluates one inclusive contiguous range of the canonical stage order;
|
||||
- defaults an omitted `--from` to `prepare` and an omitted `--through` to
|
||||
`cleanup`, so omitting both retains full-pipeline behavior;
|
||||
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
||||
- runs `render` before `extract`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
- resolves every selected typed credential during the initial configuration
|
||||
load, including a Notarius credential when extract is disabled, outside the
|
||||
selected range, or later skipped as resumable;
|
||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||
resume check finds its durable result obsolete;
|
||||
- applies `--force` only to stages in the selected range;
|
||||
- rejects repeated `--from`, `--through`, or `--force` options, including
|
||||
`--name=value` spellings;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes the session manifest as the sole durable progress ledger. Each
|
||||
invocation also creates its run-local directory and returns an immediate
|
||||
command summary; neither creates another durable progress record.
|
||||
- reports the resolved profile (or `none`) and effective configuration digest.
|
||||
|
||||
When `--artifacts` is present, the selected range must contain `analyze` or
|
||||
`publish`. Either consumer is sufficient, including a one-stage range.
|
||||
|
||||
### `regenerate-artifacts`
|
||||
|
||||
```bash
|
||||
narratio regenerate-artifacts <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Exactly equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run <session_id> --force --from extract --through analyze [caller options]
|
||||
```
|
||||
|
||||
The command always reruns extraction. Analysis rebuilds the selected configured
|
||||
artifacts and any prerequisites required by those targets; without
|
||||
`--artifacts`, it uses the normal default analysis selection. Publish and notify
|
||||
never run. Common session/configuration options and repeatable artifact values
|
||||
pass through unchanged.
|
||||
|
||||
The forced extraction rebuilds the complete configured Notarius bundle,
|
||||
disables Notarius checkpoint loading, and requests chunk-plan refresh.
|
||||
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
|
||||
|
||||
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
||||
supply those options. The shared `run` parser reports them as duplicate
|
||||
singleton flags. The alias has no private execution options or behavior, and
|
||||
runtime diagnostics may identify the operation as `run`.
|
||||
|
||||
### `run-stage`
|
||||
|
||||
```bash
|
||||
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Valid stage names:
|
||||
|
||||
- `prepare`
|
||||
- `transcribe`
|
||||
- `merge`
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `render`
|
||||
- `extract`
|
||||
- `analyze`
|
||||
- `publish`
|
||||
- `notify`
|
||||
- `cleanup`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--artifacts` is accepted only for `analyze` and `publish` stage targets.
|
||||
|
||||
### `analyze`
|
||||
|
||||
```bash
|
||||
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
This selects only `analyze`: it does not invoke Notarius, replace extraction,
|
||||
or remove existing extract outputs. `--artifacts` narrows the forced analysis
|
||||
targets and their required analysis prerequisites.
|
||||
|
||||
### `publish`
|
||||
|
||||
```bash
|
||||
narratio publish <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
This command selects only `publish`; it does not invoke cleanup. Use a normal
|
||||
full run to include configured cleanup, or `run-stage cleanup` to retry cleanup
|
||||
from valid publication evidence.
|
||||
|
||||
### `clean`
|
||||
|
||||
```bash
|
||||
narratio clean <session_id> [--dry-run] [--clear-cache] [...common config flags]
|
||||
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- session mode removes the selected session's local work and spool state;
|
||||
- `--all` removes all local session work and spool state;
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for manual and pipeline-stage
|
||||
deletion scope.
|
||||
|
||||
### `session plan`
|
||||
|
||||
```bash
|
||||
narratio session plan <session_id> [--from <stage>] [--through <stage>] [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Uses the same inclusive bounds, endpoint validation, force scope, and artifact
|
||||
selection contract as `run`. It validates config and prints run/skip decisions
|
||||
for selected stages only without creating the local workdir or changing the
|
||||
manifest. Resume-capable selected stages are checked against durable evidence.
|
||||
The output includes the resolved profile (or `none`) and effective configuration
|
||||
digest without writing provenance or any manifest state.
|
||||
For `analyze`, the preview also lists explicit targets, prerequisite-only work,
|
||||
execution order, and reusable current artifacts with concise reasons. These
|
||||
artifact decisions come from the same reconciliation and work planner used by
|
||||
execution; the preview does not predict output identities.
|
||||
|
||||
### `session validate`
|
||||
|
||||
```bash
|
||||
narratio session validate <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Read-only preflight checks for config validity, required inputs, audio mode,
|
||||
previous-session requirements, publish outputs, and independent remote locks.
|
||||
|
||||
### `session status`
|
||||
|
||||
```bash
|
||||
narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, status for the
|
||||
pointer-selected remote commit and its declared published outputs. Current
|
||||
publish metadata reports `published`, `carried_locked`, `locked_unpublished`,
|
||||
`optional_missing`, and `unselected`; carried output lines include the run
|
||||
derived from their exact declared key, the key itself, and its declared
|
||||
destination. A separately displayed `dest` is the destination requested by the
|
||||
current configuration and may differ while an older declaration is locked.
|
||||
|
||||
### `session init`
|
||||
|
||||
```bash
|
||||
narratio session init <session_id> --output ./session.yml [options]
|
||||
narratio session init <session_id> --remote [options]
|
||||
```
|
||||
|
||||
Required target selection:
|
||||
|
||||
- exactly one of:
|
||||
- `--output <path>`
|
||||
- `--remote`
|
||||
|
||||
Options:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||
- `--secrets-dir <directory>`
|
||||
- `--previous-session-id <id>`
|
||||
- `--date <YYYY-MM-DD>`
|
||||
- `--title <text>`
|
||||
- `--audio-dir <path>`
|
||||
- `--audio-s3-prefix <prefix>`
|
||||
- `--force`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--audio-dir` and `--audio-s3-prefix` are mutually exclusive.
|
||||
- if campaign `session_template_file` is configured, `session init` renders it.
|
||||
- generated session YAML must be concrete (no unresolved `{{ ... }}` placeholders).
|
||||
|
||||
### `session restore`
|
||||
|
||||
```bash
|
||||
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- discovers committed remote current state;
|
||||
- plans local restores;
|
||||
- writes an execution report;
|
||||
- blocks unresolved conflicts. `--force` permits replacement only of eligible
|
||||
regular files.
|
||||
|
||||
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
|
||||
default restore scope, report location, and conflict-handling workflow.
|
||||
|
||||
### `session artifacts`
|
||||
|
||||
```bash
|
||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
```
|
||||
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability. Remote inspection uses exact commit
|
||||
declarations and displays the five current publish outcomes when recorded.
|
||||
|
||||
### `session locks`
|
||||
|
||||
```bash
|
||||
narratio session locks <session_id> [...common config flags]
|
||||
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common config flags]
|
||||
narratio session locks remove <session_id> <source> [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode validates and reports independent remote locks sorted by source;
|
||||
- add creates one lock and reports an existing lock unless `--force` is used;
|
||||
- `--force` performs one generation-matched replacement and reports a
|
||||
concurrent change for operator retry; and
|
||||
- remove is idempotent and reports when the source was not locked.
|
||||
|
||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||
and publication behavior.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
An artifact-family key selects all of its concrete character members. A
|
||||
concrete generated key selects only that member; mixed family and concrete
|
||||
selection is deduplicated and executed as concrete keys. The resulting plan
|
||||
and command output identify both the concrete key and, where applicable, its
|
||||
family and character ID.
|
||||
|
||||
- accepted on `run`, `session plan`, `run-stage`, `analyze`, and `publish`;
|
||||
- repeatable and comma-separated values are combined, surrounding whitespace
|
||||
is removed, and duplicate names are collapsed;
|
||||
- names must exist in `pipeline.scriptorium.artifacts`;
|
||||
- empty entries are invalid;
|
||||
- on `run-stage`, only `analyze` and `publish` accept the option.
|
||||
|
||||
Effects:
|
||||
|
||||
- selects explicit analyze targets; required configured prerequisites may be
|
||||
reused or rebuilt before them;
|
||||
- filters publish rules that source `narratio.artifact.<name>`;
|
||||
- does not filter built-in transcript/bounds or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- does not select or filter Notarius lanes.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Dry-run restore plan:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Generate a concrete session file from template/default structure:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Force publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
Regenerate post-transcript artifacts without publishing:
|
||||
|
||||
```bash
|
||||
narratio regenerate-artifacts 2026-04-04 --artifacts session_recap,player_handout
|
||||
```
|
||||
|
||||
## Output And Exit Behavior
|
||||
|
||||
- Successful commands write their result or summary to standard output and
|
||||
exit with status `0`.
|
||||
- Command failures and invalid invocations write an error to standard error and
|
||||
exit with status `1`.
|
||||
- An unknown top-level command also prints the top-level usage summary to
|
||||
standard error.
|
||||
- `session restore --help` prints its command-specific usage and exits with
|
||||
status `0`.
|
||||
|
||||
Output is intended for operator inspection. Narratio does not currently offer
|
||||
a machine-readable CLI output mode; durable machine-readable state is recorded
|
||||
in manifests and reports described in [Operations](./operations.md).
|
||||
656
docs/config.md
Normal file
656
docs/config.md
Normal file
@@ -0,0 +1,656 @@
|
||||
# Configuration Reference
|
||||
|
||||
## Purpose
|
||||
|
||||
Narratio resolves three YAML documents:
|
||||
|
||||
- `pipeline.yml`: pipeline/runtime settings
|
||||
- `campaign.yml`: campaign identity and stable input defaults
|
||||
- `session.yml`: session identity, metadata, and audio source selection
|
||||
|
||||
## Discovery and Selection
|
||||
|
||||
### `pipeline.yml`
|
||||
|
||||
When `--config` is omitted, search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/pipeline.yml`
|
||||
2. `/etc/narratio/pipeline.yml`
|
||||
|
||||
### `campaign.yml`
|
||||
|
||||
Selection rules:
|
||||
|
||||
- if `--campaign-file` is set, use that path;
|
||||
- else if `--campaign <id>` is set, use `{pipeline.campaigns.root}/{id}/campaign.yml`;
|
||||
- else use `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`.
|
||||
|
||||
### `session.yml`
|
||||
|
||||
When `--session` is omitted, local search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/session.yml`
|
||||
2. `/etc/narratio/session.yml`
|
||||
|
||||
If local session discovery fails and a `session_id` is known, Narratio attempts remote session loading from:
|
||||
|
||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||
|
||||
using configured object storage.
|
||||
|
||||
The downloaded remote session file is command-scoped: Narratio removes it after
|
||||
the command finishes and records only the remote object provenance alongside
|
||||
the durable copied session input.
|
||||
|
||||
### Read-only effective pipeline inspection
|
||||
|
||||
`narratio config validate`, `narratio config show`, `narratio config sources`,
|
||||
and `narratio config artifacts` use the same `--config`, `--campaign`,
|
||||
`--campaign-file`, and
|
||||
`--profile` selection rules as pipeline commands, but do not select, discover,
|
||||
or load a session. They resolve configured typed credential references as part
|
||||
of the ordinary ConfigKit load but expose only their environment-variable names;
|
||||
they do not create runtime state.
|
||||
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
||||
campaign selectors, resolves each named profile with an independent ConfigKit
|
||||
load, and does not accept a separate `--profile` flag.
|
||||
|
||||
Campaign selection is optional only when the resolved pipeline has no
|
||||
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
||||
campaign through an explicit flag or `pipeline.campaigns.default_campaign_id`,
|
||||
then parses the campaign-owned party and expands concrete artifacts and any
|
||||
family publish rules before validation. `config validate` prints the resulting
|
||||
root, profile, and effective digest. `config show` emits the normalized
|
||||
effective pipeline YAML, with defaults and concrete expansion included but
|
||||
ConfigKit directives and family declarations omitted. This review projection is
|
||||
not promised to be a reloadable authoring document. `config sources` prints a stable
|
||||
source projection instead of effective values: root/import/profile/default
|
||||
ownership plus campaign/party and generated-family records. Canonical derived
|
||||
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
owns command syntax and output conventions.
|
||||
|
||||
`config diff` compares normalized field values rather than YAML text or source
|
||||
ownership. It emits sorted `added`, `removed`, and `changed` records, uses
|
||||
compact deterministic JSON values, treats lists atomically, and reports `no
|
||||
differences` when the complete effective configurations are equal. Concrete
|
||||
family members and generated publish rules participate after expansion; moving
|
||||
an equal value between eligible root/import sources does not create a
|
||||
difference.
|
||||
|
||||
### Migrating to the maintained bundle
|
||||
|
||||
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||
as the complete copyable migration reference. Keep production settings in the
|
||||
base, split them into explicit additive imports, and place testing differences
|
||||
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
|
||||
in the recommended one-definition files shown by that bundle. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), then express character
|
||||
work as families. Inspect the result with `config validate`, `config show`,
|
||||
`config sources`, and `config artifacts`; use `config diff default testing` to
|
||||
review the overlay before running a session. Unversioned party documents are
|
||||
rejected.
|
||||
|
||||
### Identity segments
|
||||
|
||||
Campaign IDs (`campaign_id` and `default_campaign_id`), session IDs, previous
|
||||
session IDs, and Narratio run IDs are opaque portable segments. They must use
|
||||
only ASCII letters, digits, `.`, `_`, and `-`; empty values, `.`/`..`, path
|
||||
separators, drive forms, whitespace, control characters, and non-ASCII text are
|
||||
rejected. Narratio does not trim or rewrite these values. Existing manifests or
|
||||
remote state with an unsafe legacy identity must be migrated before use.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
|
||||
unknown fields, duplicate keys, and trailing documents fail load.
|
||||
- A pipeline file may explicitly import additive YAML fragments through the
|
||||
root-only `configkit.imports` list. Imported files contribute fields to one
|
||||
logical pipeline document; they do not override fields supplied by the root
|
||||
or another import.
|
||||
- An omitted profile, an empty profile option, and the exact name `default`
|
||||
select the base configuration. Any other name selects
|
||||
`profiles/<name>.yml` relative to the pipeline root.
|
||||
- Campaign, session, and canonical-party documents are strict single-source
|
||||
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
|
||||
directives.
|
||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Session and party inputs inspected before typed loading are limited to 8 MiB.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Defaults apply only to absent fields. Explicit empty strings, zero values,
|
||||
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
|
||||
Narratio domain validation; they are not silently replaced by defaults.
|
||||
Explicit null optional stage sections stay absent, while required sections
|
||||
fail validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||
session. The optional `spell_catalog_file` also inherits a campaign value
|
||||
when omitted by the session.
|
||||
- `party_file` is parsed when pipeline and campaign configuration are combined.
|
||||
It must be a versioned [canonical party](integrations/party.md), is
|
||||
campaign-owned and derives the players input internally. `players_file` and
|
||||
session `party_file` are unsupported configuration keys.
|
||||
- Exactly one audio mode must be configured in session input:
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
|
||||
### Pipeline imports and profiles
|
||||
|
||||
Large pipeline configurations may be split into explicitly named fragments and
|
||||
may use fixed-path profile overlays:
|
||||
|
||||
```yaml
|
||||
configkit:
|
||||
imports:
|
||||
- path: config/storage.yml
|
||||
- path: config/integrations.yaml
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
```
|
||||
|
||||
The maintained [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||
is a complete copyable example of this structure, including canonical-party
|
||||
artifact families.
|
||||
|
||||
Imports are resolved relative to the directory containing the root pipeline
|
||||
file and are loaded in declaration order. Narratio does not scan directories or
|
||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
||||
import of the root pipeline itself are rejected. Imported files cannot declare
|
||||
another `configkit` control section.
|
||||
|
||||
Base merging is additive. A map may be extended by multiple files when every
|
||||
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
||||
more than once, even when the repeated values are identical. Conflict errors
|
||||
name the full field path and every source that claimed it. Narratio additionally
|
||||
rejects an artifact or artifact-family key declared by more than one base file,
|
||||
even when those files contribute disjoint children.
|
||||
|
||||
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
|
||||
rules. Only the selected overlay is opened; a malformed unselected overlay does
|
||||
not affect a base or different-profile load. Maps merge recursively, overlay
|
||||
scalars replace base scalars, and overlay lists replace base lists completely.
|
||||
Explicit `false`, zero, empty strings, empty containers, and null values remain
|
||||
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
|
||||
stack, import files, or declare another ConfigKit control section.
|
||||
|
||||
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
|
||||
entry-level defaults and normalization, read-only file-local validation, and
|
||||
ordinary path resolution before computing a deterministic effective
|
||||
configuration digest. The digest represents the normalized, secret-free
|
||||
runtime pipeline mapping; it excludes ConfigKit directives, source
|
||||
provenance, profile identity, and raw environment secret values. Equivalent
|
||||
effective mappings therefore have the same digest regardless of how fields are
|
||||
split among the root and imports.
|
||||
|
||||
An imported field has the same meaning it would have in a monolithic root
|
||||
pipeline. In particular, ordinary relative pipeline paths continue to resolve
|
||||
from the root pipeline directory, not from the importing fragment's directory.
|
||||
|
||||
## Minimal Working Configuration
|
||||
|
||||
`pipeline.yml`
|
||||
|
||||
```yaml
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
```
|
||||
|
||||
`campaign.yml`
|
||||
|
||||
```yaml
|
||||
campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
party_file: ./party.yml
|
||||
```
|
||||
|
||||
`session.yml` (local audio)
|
||||
|
||||
```yaml
|
||||
session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
```
|
||||
|
||||
## Secrets Handling
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Credential fields contain environment-variable names, represented internally
|
||||
as ConfigKit typed secret references (for example
|
||||
`pipeline.notarius.llm_api_key: NOTARIUS_LLM_API_KEY`). A configured reference
|
||||
must resolve to a nonempty environment value during load.
|
||||
- Every surviving reference is resolved during ordinary pipeline loading,
|
||||
regardless of whether its integration is disabled, outside the selected
|
||||
stage range, or eligible for resume. Omit the field or set it to `null` when
|
||||
that deployment should not require the credential.
|
||||
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
|
||||
flag is omitted, Narratio uses a present `secrets/` directory beside the
|
||||
resolved root `pipeline.yml`; an absent conventional directory is ignored.
|
||||
Explicit relative paths are resolved from the caller's working directory,
|
||||
and a missing or empty explicit selection is an error.
|
||||
- Each immediate regular file whose name is a valid environment-variable name
|
||||
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
|
||||
selected directory, rejects non-hidden symlink entries, and limits input to
|
||||
1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
|
||||
files, directories, and symlinks) are ignored and omitted from bootstrap
|
||||
reports, but still count toward the directory entry limit.
|
||||
- An existing process environment value takes precedence over a credential file.
|
||||
- On POSIX, operators must provision the directory with no group/other access
|
||||
(normally `0700`) and files with no group/other access (normally `0600`);
|
||||
ConfigKit validates type and bounds but does not enforce permission modes.
|
||||
- Bootstrap happens once per command, before any configuration load or adapter
|
||||
construction. Safe reports contain filenames, paths, and statuses, never values.
|
||||
|
||||
## Publish Configuration Summary
|
||||
|
||||
Publish rules live under `pipeline.publish`.
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `outputs[].source` is required.
|
||||
- `outputs[].dest` may be omitted when derivable from source.
|
||||
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- publish locks are independent remote session objects managed with
|
||||
`narratio session locks`; they are not pipeline configuration fields.
|
||||
|
||||
## Full Schema
|
||||
|
||||
### Pipeline
|
||||
|
||||
| Field | Type | Required | Default / Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
|
||||
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||
| `pipeline.publish.upload_run` | bool | No | `true` |
|
||||
| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed JSON plus final and final-trimmed Markdown outputs |
|
||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||
| `pipeline.whisperx.retry_delay` | duration | No | `2s` |
|
||||
| `pipeline.whisperx.concurrency` | int | No | `2` |
|
||||
| `pipeline.seriatim.binary` | string | No | `seriatim` |
|
||||
| `pipeline.seriatim.timeout` | duration | No | `10m` |
|
||||
| `pipeline.seriatim.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.seriatim.coalesce_gap` | float | No | `3.0` |
|
||||
| `pipeline.seriatim.report` | bool | No | `true` |
|
||||
| `pipeline.seriatim.env.overlap_word_run_gap` | float | No | unset |
|
||||
| `pipeline.seriatim.env.overlap_word_run_reorder_window` | float | No | unset |
|
||||
| `pipeline.seriatim.env.backchannel_max_duration` | float | No | unset |
|
||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||
| `pipeline.audita.binary` | string | No | `audita` |
|
||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||
| `pipeline.audita.llm_api_key` | secret reference | No | unset |
|
||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||
| `pipeline.audita.base_url` | string | No | empty |
|
||||
| `pipeline.audita.model` | string | No | empty |
|
||||
| `pipeline.audita.total_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.proposal_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.validation_model` | string | No | empty |
|
||||
| `pipeline.audita.validation_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.transcript_description` | string | No | empty |
|
||||
| `pipeline.audita.config_path` | string | No | empty |
|
||||
| `pipeline.audita.output_schema` | string | No | empty |
|
||||
| `pipeline.audita.work_dir_retention` | string | No | empty |
|
||||
| `pipeline.audita.report` | bool | No | `true` |
|
||||
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
|
||||
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.normalize.report` | bool | No | `true` |
|
||||
| `pipeline.trim.enabled` | bool | No | `true` |
|
||||
| `pipeline.trim.output_path` | string | No | `transcripts/final.trimmed.json` |
|
||||
| `pipeline.trim.bounds.prompt_id` | string | No | `dnd.session_bounds` |
|
||||
| `pipeline.trim.bounds.profile_id` | string | No | empty |
|
||||
| `pipeline.trim.bounds.transcript_input_name` | string | No | `transcript` |
|
||||
| `pipeline.trim.bounds.output_path` | string | No | `artifacts/session_bounds.json` |
|
||||
| `pipeline.trim.bounds.timeout` | duration | No | `10m` |
|
||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
|
||||
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
|
||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
|
||||
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
|
||||
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||
| `pipeline.render.enabled` | bool | No | `true` |
|
||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||
| `pipeline.render.include_timestamps` | bool | No | `true` |
|
||||
| `pipeline.render.include_segment_ids` | bool | No | `true` |
|
||||
| `pipeline.render.include_metadata` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.binary` | string | No | `scriptorium` |
|
||||
| `pipeline.scriptorium.config_path` | string | No | empty |
|
||||
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
||||
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||
|
||||
### Notarius Configuration Profile
|
||||
|
||||
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
|
||||
configuration profile when extraction executes:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
config_profile: downstream-economy
|
||||
```
|
||||
|
||||
When the field is omitted or `null`, Narratio supplies no downstream profile
|
||||
selector and preserves Notarius's existing configuration-selection behavior.
|
||||
When present, Narratio forwards the exact string to Notarius, including an
|
||||
empty string or `default`, which explicitly select Notarius's composed base
|
||||
configuration. Notarius owns selector interpretation and validation; Narratio
|
||||
does not restrict profile names, resolve profile files, or rewrite the value.
|
||||
|
||||
This selection is independent of Narratio's own `--profile` selection. A
|
||||
Narratio overlay may explicitly select a differently named Notarius overlay,
|
||||
but Narratio never forwards its own profile name automatically. It is also
|
||||
independent of `pipeline.notarius.llm_api_key`: an overlay may change either or
|
||||
both fields according to the deployed Notarius configuration. See the
|
||||
[Notarius integration contract](./integrations/notarius.md) for invocation and
|
||||
failure behavior.
|
||||
|
||||
### Notarius Credential
|
||||
|
||||
`pipeline.notarius.llm_api_key` optionally names the one environment variable
|
||||
to forward to the Notarius child process:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
llm_api_key: OPENROUTER_API_KEY
|
||||
```
|
||||
|
||||
The field contains the variable name, not a credential value. That name must
|
||||
match `api_key_env` in the Notarius/PromptKit profile selected by the deployed
|
||||
Notarius pipeline; Narratio does not infer or remap provider names. Omission or
|
||||
`null` forwards nothing. A non-null selected reference must resolve at pipeline
|
||||
load time through the process environment or existing secrets-directory
|
||||
bootstrap, even when Notarius is disabled, extract is outside the requested
|
||||
range, or a completed extraction can be reused.
|
||||
|
||||
The Notarius child receives only Narratio's minimal execution environment plus
|
||||
this exact selected name and loaded value. Other credentials are not forwarded.
|
||||
Changing the value, name, presence, or absence does not invalidate extract's
|
||||
semantic resume evidence. The public name remains part of safe effective
|
||||
inspection, so renaming it may appear in `config diff` and change the effective
|
||||
configuration digest; rotating only its value does neither.
|
||||
|
||||
See the [Notarius integration contract](./integrations/notarius.md) for the
|
||||
delivery and redaction boundary.
|
||||
|
||||
### Notarius Cache Controls
|
||||
|
||||
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
|
||||
extraction passes `--resume`; the deployed Notarius configuration must already
|
||||
enable checkpoint recording and should select an explicit persistent checkpoint
|
||||
root. Narratio does not inspect or generate that configuration.
|
||||
|
||||
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
|
||||
configured mode in control. A nonempty value must be `auto`, `bypass`, or
|
||||
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
|
||||
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
|
||||
Neither field changes Narratio's semantic extraction fingerprint, so changing
|
||||
them alone does not rerun an otherwise complete durable extraction.
|
||||
|
||||
### Notarius Reference Bindings
|
||||
|
||||
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
||||
Narratio source, not to a filesystem path:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
references:
|
||||
glossary: narratio.input.glossary
|
||||
party: narratio.input.party
|
||||
players: narratio.input.players
|
||||
spell_catalog: narratio.input.spell_catalog
|
||||
```
|
||||
|
||||
Supported sources are `narratio.input.party`, `narratio.input.players`,
|
||||
`narratio.input.glossary`, and `narratio.input.spell_catalog`. Each map entry is
|
||||
required by its presence: omit a binding when the selected Notarius pipeline
|
||||
does not need it. A spell-catalog binding additionally requires an effective
|
||||
campaign or session `spell_catalog_file`.
|
||||
|
||||
Selectors accept Notarius's `slot`, `chunk.slot`, `lane.slot`,
|
||||
`lane.extract.slot`, `lane.merge.slot`, and `lane.normalize.slot` forms.
|
||||
Narratio trims whitespace around
|
||||
selectors and their dot-separated components, rejects empty components and
|
||||
`=`, rejects duplicate normalized selectors, and limits the map to 256 entries.
|
||||
It validates only selector structure and the prepared source vocabulary;
|
||||
Notarius owns target-slot declarations and media compatibility.
|
||||
|
||||
Before extraction, Narratio resolves every binding from the current prepared
|
||||
session manifest and streams it into a verified, content-addressed session
|
||||
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
|
||||
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
||||
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
||||
part of extraction fingerprint and resume identity. See the
|
||||
[Notarius integration contract](./integrations/notarius.md) for the subprocess
|
||||
boundary and the [complete example](../examples/pipeline.full.annotated.yml)
|
||||
for a copyable configuration.
|
||||
|
||||
### Notarius Output Entries
|
||||
|
||||
For each `pipeline.notarius.outputs.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||
| `module_key` | string | No | exact accepted module key when set |
|
||||
|
||||
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||
contract-incompatible lane fails extraction. See the
|
||||
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||
for compatibility ownership.
|
||||
|
||||
### Scriptorium Artifact Entries
|
||||
|
||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | bool | No | `false` if omitted |
|
||||
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
|
||||
| `render_debug` | bool | No | per-artifact override |
|
||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||
| `profile_id` | string | No | empty |
|
||||
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
|
||||
| `timeout` | duration | No | artifact override |
|
||||
| `inputs` | map | No | input key names must be non-empty |
|
||||
| `vars` | map | No | values must be string or bool; `session_id` is reserved and overwritten by Narratio |
|
||||
|
||||
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
|
||||
|
||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||
those targets over the effective dependency graph: a current prerequisite is reused, while a
|
||||
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||
may require rebuilding must therefore have valid executable fields. This
|
||||
override affects analyze planning only; publish uses the list only to filter
|
||||
configured `narratio.artifact.<name>` output rules.
|
||||
|
||||
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `required` | bool | No | optional input requirement |
|
||||
|
||||
`artifact` and `path` are obsolete and rejected by strict configuration
|
||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||
does not provide adapter-specific input passthrough fields.
|
||||
|
||||
Each `narratio.artifact.<name>` input source automatically contributes an
|
||||
effective dependency on that configured artifact. Narratio unions these inferred
|
||||
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
|
||||
result during final configuration resolution. Authors use `depends_on` only for
|
||||
ordering constraints that are not already expressed by an artifact input.
|
||||
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
to inspect keys, definition sources, outputs, and effective dependencies without
|
||||
loading a session or creating runtime state.
|
||||
|
||||
### Scriptorium Artifact Families
|
||||
|
||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||
for every canonical campaign character. Configuration resolution expands each
|
||||
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
|
||||
planning or Scriptorium invocation. Party parsing rejects unsupported document
|
||||
schemas before family expansion.
|
||||
|
||||
For each `pipeline.scriptorium.artifact_families.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled`, `prompt_id`, `profile_id`, `timeout`, `render_debug`, `depends_on`, `inputs`, `vars` | ordinary artifact fields | No | copied to each generated artifact under the corresponding ordinary rules |
|
||||
| `for_each` | string | Yes | exactly `party.characters` |
|
||||
| `output_path_pattern` | string | Yes | safe path beneath `artifacts/` with exactly one `{character_id}` token and no other brace syntax |
|
||||
| `member_vars` | map | No | maps an ordinary Scriptorium variable name to a supported canonical character selector |
|
||||
| `member_dependencies` | list | No | unique family keys; each generated member depends on the corresponding generated member of each listed family |
|
||||
| `publish` | map | No | typed family publish policy (`enabled`, `required`, `dest_pattern`) expanded into concrete publish outputs when enabled |
|
||||
|
||||
Generated keys are `<family>_<character_id>` and generated output paths must
|
||||
not collide with explicit artifacts or another generated artifact. Families
|
||||
expand even when disabled; normal analyze selection still omits disabled
|
||||
artifacts unless they are explicitly selected by their concrete key.
|
||||
|
||||
Supported `member_vars` selectors are `character_id`, `player.name`,
|
||||
`character.name`, `character.class_summary`, and `character.alias_summary`.
|
||||
Their resolved values are strings. A member variable may not reuse a static
|
||||
`vars` name; `session_id` remains owned and overwritten by Narratio as for any
|
||||
other Scriptorium artifact.
|
||||
|
||||
Within a family only, an input source may use
|
||||
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
|
||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
|
||||
infers the concrete member dependency. Use `member_dependencies` only for
|
||||
additional same-character ordering constraints that are not already inputs.
|
||||
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
||||
or Scriptorium.
|
||||
|
||||
### Notifications
|
||||
|
||||
Narratio currently supports only `notification.mode: noop`, which is also the
|
||||
default when the section is omitted. The notify stage performs no delivery in
|
||||
this mode. Backend, recipient, timeout, and other provider settings are
|
||||
rejected by strict configuration loading until Narratio has a provider
|
||||
integration.
|
||||
|
||||
### Campaign
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `campaign_id` | string | Yes | canonical opaque campaign identity |
|
||||
| `session_template_file` | string | No | used by `session init` when set |
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
|
||||
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
|
||||
|
||||
### Session
|
||||
|
||||
| Field | Type | Required in session file | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `session_id` | string | Yes | opaque identity; must match CLI session target when provided |
|
||||
| `previous_session_id` | string | No | opaque identity; must not equal `session_id` |
|
||||
| `campaign` | string | No | opaque identity; filled from `campaign_id` during resolve if omitted |
|
||||
| `date` | string | No | metadata |
|
||||
| `title` | string | No | metadata |
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
|
||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
|
||||
|
||||
Audio rules:
|
||||
|
||||
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
- `audio_s3` requires `pipeline.storage.backend: s3` and a configured S3 bucket.
|
||||
|
||||
### Storage backend selection
|
||||
|
||||
`local` is the default and disables remote object-store operations. Configure
|
||||
`s3` explicitly before supplying `storage.s3`; a populated S3 block does not
|
||||
select a backend on its own. Unknown backend names and an S3 block paired with
|
||||
`local` are rejected during configuration validation.
|
||||
|
||||
### Previous-session expectation
|
||||
|
||||
`previous_session_id` is optional in a session file. When a command supplies
|
||||
`--previous-session-id`, however, the session file must contain the same value;
|
||||
an omitted or different value is rejected before the command performs work.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
See the [maintained examples index](../examples/README.md) for complete pipeline,
|
||||
campaign, session, template, and input fixtures. Keep complete copyable files
|
||||
there rather than duplicating them in this reference.
|
||||
64
docs/development.md
Normal file
64
docs/development.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Development
|
||||
|
||||
This is the first-read landing page for people and LLM coding agents working on
|
||||
Narratio. It provides a concise repository orientation and routes each kind of
|
||||
change to its canonical documentation.
|
||||
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts and generated artifacts. Start with the
|
||||
[README](../README.md) for product context,
|
||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||
|
||||
## Go Prerequisite
|
||||
|
||||
Maintained development and verification require Go 1.26. Use the official Go
|
||||
1.26 toolchain before running repository commands.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
| --- | --- | --- |
|
||||
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internal documents. |
|
||||
| Application shape, boundaries, dependency direction, runtime invariants, safety properties, or dependencies | [Architecture](policy/architecture.md) | It defines the intended system shape, ownership, and non-goals. |
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical owners, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Adding, changing, reviewing, rewriting, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and test lifecycle decisions. |
|
||||
| CLI composition or command behavior | [Internal Overview](internal/overview.md) and [CLI Reference](cli.md) | The overview routes to command ownership; the reference owns public syntax and invocation behavior. |
|
||||
| Configuration loading, resolution, or user-visible configuration | [Internal Overview](internal/overview.md) and [Configuration](config.md) | The overview routes to implementation ownership; the reference owns fields, defaults, discovery, and validation. |
|
||||
| Session workflow, status, restore, cleanup, or object storage | [Restore Internals](internal/command-restore.md), [Workspace Internals](internal/workspace.md), [Storage Internals](internal/storage.md), [Operations](operations.md), and [Troubleshooting](troubleshooting.md) | These separate implementation mechanics, operator procedures, and symptom-driven recovery. |
|
||||
| Pipeline sequencing or the behavior of a stage | [Internal Overview](internal/overview.md) and its focused stage documents | The overview owns the implemented stage inventory and routes to each stage contract. |
|
||||
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
|
||||
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
|
||||
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
|
||||
| Preparing, validating, or publishing a release | [Release Procedure](release.md) | The maintainer procedure owns version selection, candidate validation, guarded tag publication, and optional later CI inspection. |
|
||||
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
For an existing subsystem, also inspect its focused tests and package-level
|
||||
contracts before changing behavior.
|
||||
|
||||
## Validation
|
||||
|
||||
Use focused package tests while iterating. Every pull request and push runs the
|
||||
following repository-wide checks before it can be accepted:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go test ./internal/doccheck
|
||||
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
```
|
||||
|
||||
The documentation check verifies local Markdown links and the dependency graph
|
||||
of the Woodpecker workflows. The configuration check loads every maintained
|
||||
pipeline and session example. Tag CI reuses this validation path before its
|
||||
asynchronous asset publication; the maintainer release boundary is documented
|
||||
in the [Release Procedure](release.md).
|
||||
|
||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
|
||||
target. The macOS cross-build establishes compilation only, not native runtime
|
||||
evidence. Add a native check only when an official runner label and successful
|
||||
native-run evidence are available.
|
||||
36
docs/integrations/README.md
Normal file
36
docs/integrations/README.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Integrations Index
|
||||
|
||||
## Audience
|
||||
|
||||
Operators, developers, and coding agents who need to understand Narratio's
|
||||
externally observable integration boundaries.
|
||||
|
||||
## Scope
|
||||
|
||||
`docs/integrations/` is the canonical reference for protocols, invocation and
|
||||
data contracts, logical outputs, and compatibility behavior at external tool
|
||||
boundaries.
|
||||
|
||||
These documents describe what Narratio sends or invokes, what it accepts in
|
||||
return, and how failures are surfaced. Internal composition and stage mechanics
|
||||
belong in [the adapter implementation guide](../internal/adapters.md) and the
|
||||
focused stage documents.
|
||||
|
||||
## Integration Contracts
|
||||
|
||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||
discovery (`notarius run`).
|
||||
- [Party](./party.md): canonical campaign roster input.
|
||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||
(`scriptorium run|render`).
|
||||
- [WhisperX](./whisperx.md): speaker-audio transcription over HTTP.
|
||||
|
||||
## Related Canonical Docs
|
||||
|
||||
- [Configuration](../config.md): operator-facing configuration reference.
|
||||
- [Adapter implementation](../internal/adapters.md): shared adapter boundary and
|
||||
runner wiring.
|
||||
- [Internal documentation](../internal/overview.md): stage-specific integration
|
||||
usage and component ownership.
|
||||
@@ -1,96 +1,69 @@
|
||||
# Audita Subprocess Operations
|
||||
# Integration: Audita
|
||||
|
||||
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
||||
## Purpose
|
||||
Define the Audita adapter contract used by the `polish` stage.
|
||||
|
||||
## Recommended command form
|
||||
## External Boundary
|
||||
|
||||
Use explicit file outputs for orchestrated runs:
|
||||
Narratio invokes `audita process` as a subprocess for each polish operation.
|
||||
The configured timeout and parent cancellation bound the invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
## Request Contract
|
||||
`PolishRequest` carries:
|
||||
- required transcript/glossary/output/work-dir paths;
|
||||
- optional report path (required when report mode is enabled);
|
||||
- generated config and stdout/stderr log paths;
|
||||
- optional per-invocation module override.
|
||||
|
||||
Additional flags that may be situationally appropriate:
|
||||
- `--config <path>` to select an explicit versioned config file.
|
||||
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
||||
- `--work-dir <dir>` to control diagnostics location.
|
||||
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
||||
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
||||
- `--modules ...` only when intentionally overriding the default sequence.
|
||||
The constructed runner owns static Audita settings: binary, timeout,
|
||||
credentials, default modules, model and endpoint settings, validation and output
|
||||
settings, report mode, and concurrency. The `polish` stage supplies only
|
||||
invocation-specific paths and may override modules for that invocation.
|
||||
|
||||
For config-driven orchestration, validate config files in CI/preflight:
|
||||
## Result Contract
|
||||
`PolishResult` returns:
|
||||
- processed transcript path;
|
||||
- optional report path;
|
||||
- work dir and generated-config/log paths;
|
||||
- exit code, duration, binary provenance;
|
||||
- adapter metadata map.
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
```
|
||||
## Validation and Failure Semantics
|
||||
Construction fails for invalid static config values, including:
|
||||
- empty binary;
|
||||
- non-positive timeout;
|
||||
- invalid base URL;
|
||||
- invalid output schema;
|
||||
- invalid work-dir retention value;
|
||||
- invalid concurrency values.
|
||||
|
||||
## Stdout behavior
|
||||
Run fails for:
|
||||
- missing required request paths;
|
||||
- subprocess execution failure;
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- Report JSON is never written to stdout.
|
||||
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
|
||||
MiB. Both must be regular files without symlinked path components.
|
||||
|
||||
## Stderr behavior
|
||||
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
- Success path should be quiet or minimal human-readable logs.
|
||||
- Failure path writes concise human-readable errors.
|
||||
- When a diagnostics run directory exists, failure stderr includes its path.
|
||||
- Prompt/response diagnostic payloads are not streamed to stderr.
|
||||
## Deterministic Behavior
|
||||
- CLI args are built from runner config + request in a fixed order.
|
||||
- A configured typed `llm_api_key` reference is resolved during pipeline load;
|
||||
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
|
||||
override and is never re-read from the parent environment.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
## Output file behavior
|
||||
## Configuration
|
||||
|
||||
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
||||
- Output write failures return nonzero and surface actionable errors.
|
||||
- The command does not silently ignore output write errors.
|
||||
Operator-selected values are defined under `pipeline.audita.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
## Report JSON behavior
|
||||
Maintained example with Audita config:
|
||||
|
||||
- `--report-json` writes a machine-readable process report to the requested path.
|
||||
- Run-directory `report.json` is written independently under diagnostics.
|
||||
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
||||
- Report write failures return nonzero with clear stderr messaging.
|
||||
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
|
||||
- Each run creates (when possible) a per-run diagnostics directory.
|
||||
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
||||
- Failed runs retain diagnostics.
|
||||
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
||||
|
||||
## Exit codes
|
||||
|
||||
- `0`: success.
|
||||
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
||||
|
||||
Treat any nonzero as a failed subprocess invocation.
|
||||
|
||||
## Timeout and cancellation
|
||||
|
||||
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
||||
- On cancellation or timeout, the process exits nonzero and should not hang.
|
||||
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
||||
|
||||
## Secret redaction expectations
|
||||
|
||||
API keys and configured secret values are redacted from:
|
||||
- reports (`--report-json` and run-dir `report.json`);
|
||||
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
||||
- surfaced adapter/runtime errors;
|
||||
- test fixtures and regression outputs.
|
||||
|
||||
Parent-process logs should still avoid printing raw environment variables.
|
||||
|
||||
## Parent-process pipe guidance
|
||||
|
||||
To avoid deadlocks in orchestrators:
|
||||
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
||||
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
||||
- treat stderr as human-readable diagnostics, not structured data;
|
||||
- parse structured results from output/report files.
|
||||
|
||||
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
200
docs/integrations/notarius.md
Normal file
200
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,200 @@
|
||||
# Notarius Integration Contract
|
||||
|
||||
## Boundary
|
||||
|
||||
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||
|
||||
Canonical Notarius v0.7.0 references:
|
||||
|
||||
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
|
||||
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
||||
|
||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||
Notarius documents as canonical when changing those values; Narratio does not
|
||||
duplicate the complete schemas.
|
||||
|
||||
## Invocation
|
||||
|
||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||
configuration path, input path, output directory, and working directory to
|
||||
absolute paths. Configured references require support for the `--reference`
|
||||
contract introduced in Notarius v0.6.0. Cache controls require support for the
|
||||
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
|
||||
cache integration baseline, not an unconditional minimum version; Narratio
|
||||
does not impose a runtime version gate. Each reference binding is a separate
|
||||
argument before `--json`. When `pipeline.notarius.config_profile` is present,
|
||||
its exact value is another argument pair before `--json`:
|
||||
|
||||
```text
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
|
||||
```
|
||||
|
||||
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
|
||||
option requires a compatible Notarius executable, but deployments that omit the
|
||||
field acquire no additional version requirement. Omitted or null configuration
|
||||
adds no flag. Supplied empty, `default`, and named selectors are forwarded
|
||||
without normalization so Notarius can apply its own ConfigKit semantics. This
|
||||
option selects a Notarius configuration overlay, not a PromptKit profile through
|
||||
`--llm-profile`, and Narratio never substitutes its own selected profile name.
|
||||
A missing or invalid downstream overlay is an ordinary Notarius subprocess
|
||||
failure: Narratio does not retry without the selector or fall back to another
|
||||
profile.
|
||||
|
||||
Reference paths are absolute stable snapshots streamed from manifest-verified
|
||||
canonical files prepared inside the current Narratio session workspace. They
|
||||
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
|
||||
full lowercase content digest. Narratio verifies snapshot checksum and size
|
||||
before and after the subprocess, reconstructs missing or corrupt regular files
|
||||
from a still-valid prepared source, and passes only configured bindings,
|
||||
ordered lexically by normalized selector, as direct argument-vector entries
|
||||
without shell interpretation. Symlinks, directories, other non-regular
|
||||
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
|
||||
binding takes precedence over a matching external path in Notarius
|
||||
configuration. Narratio never emits `--without-reference`.
|
||||
|
||||
For canonical party configuration, the `party` binding is the unchanged,
|
||||
validated authored roster and the `players` binding is its generated
|
||||
projection. Both retain their established `narratio.input.party` and
|
||||
`narratio.input.players` source IDs, and both are resolved from the prepared
|
||||
manifest rather than from campaign configuration at extraction time.
|
||||
|
||||
The maintained D&D boundary binds only the four campaign-owned external slots:
|
||||
|
||||
```text
|
||||
notarius run dnd-session \
|
||||
--config <absolute config path> \
|
||||
--input <absolute trimmed transcript path> \
|
||||
--output-dir <absolute staging directory> \
|
||||
--reference glossary=<absolute verified glossary snapshot> \
|
||||
--reference party=<absolute verified party snapshot> \
|
||||
--reference players=<absolute verified players snapshot> \
|
||||
--reference spell_catalog=<absolute verified spell catalog snapshot> \
|
||||
--json
|
||||
```
|
||||
|
||||
The spell-catalog binding is omitted when the campaign does not maintain that
|
||||
optional overlay. Registry, scene-description, combat-turn, and occurrence
|
||||
handoffs generated during the same Notarius run remain in Notarius pipeline
|
||||
composition and must not be emitted as CLI references. The linked CLI and D&D
|
||||
consumer documents own selector targeting, declared slots, media compatibility,
|
||||
and generated-handoff collision rules.
|
||||
|
||||
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||
separately as diagnostic output. Narratio applies the configured timeout and
|
||||
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||
It does not pass a Narratio session ID or run `notarius config validate`
|
||||
automatically; the configured working directory and Narratio's minimal child
|
||||
environment apply to the subprocess.
|
||||
|
||||
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
|
||||
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
|
||||
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
|
||||
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
|
||||
Narratio neither reads nor synthesizes its configuration. Use explicit
|
||||
persistent roots in Notarius when reuse must span subprocess invocations.
|
||||
Changing a reference path/digest or other upstream identity input correctly
|
||||
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
|
||||
replacement for the active source; it does not delete unrelated plans or
|
||||
checkpoint state.
|
||||
|
||||
When `pipeline.notarius.llm_api_key` is absent or null, Narratio adds no
|
||||
credential to that environment. When configured, the field names an environment
|
||||
variable whose already-loaded nonempty value is forwarded under that exact
|
||||
name. Operators must use the same name as the `api_key_env` in the selected
|
||||
Notarius/PromptKit profile. Narratio does not inspect the Notarius configuration
|
||||
to infer a provider or destination name, and it does not forward unrelated
|
||||
credentials from its parent environment or secrets directory.
|
||||
|
||||
The selected name is registered with Narratio's shared streaming redactor.
|
||||
Credential bytes written by the child to captured stdout, captured stderr, or
|
||||
stderr-derived failure diagnostics are replaced with `<redacted>`, including
|
||||
custom names not known to the built-in sensitive-name list. Narratio does not
|
||||
serialize the credential into arguments, stage metadata, manifests, or
|
||||
generated configuration.
|
||||
|
||||
This captured-stream protection is not a filesystem sandbox. Notarius remains
|
||||
responsible for keeping credentials out of files it writes directly, including
|
||||
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
|
||||
arbitrary downstream files and cannot promise redaction of transformed or
|
||||
encoded credentials.
|
||||
|
||||
## Accepted Result
|
||||
|
||||
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
|
||||
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
||||
receipt or ten-lane output contract. The receipt
|
||||
must identify the configured pipeline, and its `index_file` must be exactly
|
||||
`index.json` beneath the reported bundle root. The production index must name
|
||||
the management files exactly as `manifest.json`, `rejected.json`,
|
||||
`warnings.json`, and `diagnostics.json`. All receipt, index, and lane paths must
|
||||
stay inside that bundle; symlinks and non-regular lane payloads are rejected.
|
||||
|
||||
Supported receipt and index shapes tolerate unknown fields for forward
|
||||
compatibility, while required identity, validation, count, manifest,
|
||||
rejection, warning, diagnostic, and lane-list fields remain mandatory.
|
||||
Narratio applies bounded reads to the receipt, index, rejection, warning, and
|
||||
diagnostic documents. Warning and diagnostic envelopes, group counts,
|
||||
occurrence counts, truncation state, framework-owned origins, and
|
||||
receipt-to-bundle counts must be internally consistent. Optional chunk-map and
|
||||
evidence-context descriptors must carry their complete generic contract
|
||||
metadata when present.
|
||||
|
||||
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||
incompatible required lanes fail extraction even if Notarius exited zero. A
|
||||
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
||||
fails extraction.
|
||||
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
|
||||
preserves that valid summary rather than treating it as a missing field.
|
||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||
selectable Narratio sources.
|
||||
|
||||
Each accepted configured lane is registered as
|
||||
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||
|
||||
## Failure And Compatibility Behavior
|
||||
|
||||
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||
staged bundle is promoted to durable storage.
|
||||
- Contract and external provenance metadata are preserved on lane artifact
|
||||
records and through explicit publication.
|
||||
- Undeclared selectors, incompatible reference files, and external/generated
|
||||
reference collisions are Notarius errors and fail extraction normally.
|
||||
|
||||
Rejection, validation, warning, and diagnostic summaries retain bounded stable
|
||||
identity, category, origin, reason-code, status, and occurrence fields without
|
||||
copying free-form external messages into Narratio manifest metadata or reading
|
||||
lane payload bodies.
|
||||
|
||||
Configuration fields and defaults are in [Configuration](../config.md).
|
||||
Operator paths, rerun procedures, and bundle retention are in
|
||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||
for failure recovery.
|
||||
|
||||
## Opt-in Integration Test
|
||||
|
||||
The production-boundary cache test requires an absolute path to a locally
|
||||
built Notarius v0.7.0-compatible executable. It uses only a local
|
||||
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
|
||||
|
||||
```sh
|
||||
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
|
||||
GOWORK=off go test ./internal/stage \
|
||||
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
|
||||
```
|
||||
|
||||
When the variable is unset, the normal test suite skips this test and never
|
||||
searches `PATH`, a sibling checkout, the network, or a real provider.
|
||||
71
docs/integrations/party.md
Normal file
71
docs/integrations/party.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Canonical Party Input
|
||||
|
||||
`party.yml` is a campaign-owned roster input. It must be a single, versioned
|
||||
`narratio.party.v1` document when Narratio resolves a pipeline, campaign, and
|
||||
session together.
|
||||
|
||||
```yaml
|
||||
schema_version: narratio.party.v1
|
||||
|
||||
characters:
|
||||
arannis:
|
||||
player:
|
||||
name: Eric
|
||||
character:
|
||||
name: Arannis
|
||||
alias:
|
||||
- Ari
|
||||
- The Grey Owl
|
||||
classes:
|
||||
- name: wizard
|
||||
level: 8
|
||||
```
|
||||
|
||||
`characters` is a non-empty mapping. Each key is a stable character ID using
|
||||
the configured-artifact key grammar: a lowercase ASCII letter followed by zero
|
||||
or more lowercase ASCII letters, digits, or underscores. Character order is
|
||||
preserved where roster order matters.
|
||||
|
||||
Every entry has `player.name`, `character.name`, and a non-empty
|
||||
`character.classes` list. Class entries require a non-empty `name` and may
|
||||
include a positive integer `level`. The optional, intentionally singular
|
||||
`character.alias` field is a list. Names, aliases, and class names must be
|
||||
non-empty, trimmed display strings without control characters. Character names
|
||||
and aliases must be unique across the full roster under Unicode-aware
|
||||
case-insensitive comparison; player names may repeat.
|
||||
|
||||
The document has exactly one YAML document and accepts no unknown fields.
|
||||
`schema_version` is required: a missing, wrong, or malformed value is an error.
|
||||
|
||||
## Migration
|
||||
|
||||
Unversioned party inputs are rejected. Replace each legacy party document with
|
||||
the versioned structure above, move each roster entry under `characters`, and
|
||||
remove separate player and session-party settings: those keys are unsupported.
|
||||
|
||||
The configuration rules and source-relative path behavior are defined in the
|
||||
[Configuration Reference](../config.md).
|
||||
|
||||
## Derived players document
|
||||
|
||||
During `prepare`, Narratio copies the canonical party source bytes unchanged
|
||||
to `inputs/party.yml` and writes this deterministic players-only projection to
|
||||
`inputs/players.yml`:
|
||||
|
||||
```yaml
|
||||
schema_version: narratio.players.v1
|
||||
players:
|
||||
- name: Eric
|
||||
character:
|
||||
id: arannis
|
||||
name: Arannis
|
||||
alias:
|
||||
- Ari
|
||||
- The Grey Owl
|
||||
```
|
||||
|
||||
There is one entry per character, sorted by stable character ID. Repeated
|
||||
player names remain separate entries. The optional `alias` list retains its
|
||||
declared order and is omitted when empty. The projection carries no class
|
||||
data. Its prepared manifest record is marked `derived_from_party`; it is not a
|
||||
separate user-provided `players_file`.
|
||||
@@ -1,339 +1,71 @@
|
||||
# Narratio -> Scriptorium CLI Integration
|
||||
# Integration: Scriptorium
|
||||
|
||||
## 1. Purpose
|
||||
## Purpose
|
||||
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
|
||||
|
||||
This document defines how Narratio should invoke Scriptorium through the **public CLI**.
|
||||
## External Boundary
|
||||
|
||||
This is a **subprocess integration contract**, not an internal Go API contract.
|
||||
|
||||
## 2. Assumptions
|
||||
|
||||
- `scriptorium` is installed and available on `PATH`.
|
||||
- Scriptorium is configured with `config.yml`.
|
||||
- `config.yml` provides `prompt_dir`, `profile_dir`, and `schema_dir` as needed.
|
||||
- Prompt and profile libraries are already deployed for the environment.
|
||||
- Narratio provides prepared artifact files (for example polished transcript, glossary, previous recap, campaign notes).
|
||||
- Initial integration is synchronous subprocess execution.
|
||||
- Narratio remains the orchestrator.
|
||||
|
||||
In normal operation, Narratio does not need to pass `--prompt-dir` and `--profile-dir` if they are supplied by Scriptorium config.
|
||||
|
||||
Narratio may pass `--config <PATH>` when it must use a non-default Scriptorium config file.
|
||||
|
||||
## 3. Core Commands Narratio May Call
|
||||
|
||||
Primary commands for subprocess integration:
|
||||
Narratio invokes Scriptorium as a subprocess in these modes:
|
||||
|
||||
- `scriptorium run`
|
||||
- `scriptorium render`
|
||||
|
||||
For production generation, use `scriptorium run`.
|
||||
|
||||
`scriptorium render` is for debugging, dry-runs, test assertions, and validating command construction without LLM execution.
|
||||
|
||||
Note: `scriptorium serve` and HTTP API exist, but they are not the initial integration path.
|
||||
|
||||
## 4. Command Selection Guidance
|
||||
|
||||
- Use `run` to generate an output artifact.
|
||||
- Use `render` to inspect the prepared prompt and effective settings without calling the LLM.
|
||||
- Use `render --format json` when Narratio/tests need structured prepare output.
|
||||
|
||||
## 5. Recommended `run` Invocation Shape
|
||||
|
||||
Production shape:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--out <output-artifact-path>
|
||||
```
|
||||
|
||||
Common optional additions:
|
||||
|
||||
- `--config <path>`: use a specific Scriptorium config file.
|
||||
- `--profile <profile_id>`: override prompt default profile.
|
||||
- `--var name=value` (repeatable): small metadata values.
|
||||
- `--input name=path` (repeatable): additional named artifacts.
|
||||
- `--timeout <duration>`: per-run timeout override.
|
||||
- Runtime model override flags (`--llm-base-url`, `--model`, etc.) only for exceptional/operator-directed cases.
|
||||
|
||||
## 6. Recommended `render` Invocation Shape
|
||||
|
||||
Human-readable debug shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format text
|
||||
```
|
||||
|
||||
Structured debug/test shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format json \
|
||||
--out <render-debug-path>
|
||||
```
|
||||
|
||||
`render` does **not** call the LLM, does **not** validate model output, and does **not** perform repair.
|
||||
|
||||
## 7. Inputs
|
||||
|
||||
- Pass inputs as repeated `--input name=path` flags.
|
||||
- `name` must match the Prompt Definition input name.
|
||||
- Prefer absolute paths, or paths relative to a working directory controlled by Narratio.
|
||||
- Pass Audita output as the primary transcript input.
|
||||
- Additional inputs may include glossary, previous recap, campaign notes, event logs, final state maps, or other prompt-specific artifacts.
|
||||
- Scriptorium reads input files directly; Narratio does not need to inline file content for CLI use.
|
||||
|
||||
## 8. Variables
|
||||
|
||||
Use repeated `--var name=value` for small metadata values.
|
||||
|
||||
Typical examples:
|
||||
|
||||
- `session_date`
|
||||
- `session_id`
|
||||
- `campaign_name`
|
||||
- `previous_session_id`
|
||||
- `output_kind`
|
||||
|
||||
Large content belongs in input files, not `--var` values.
|
||||
|
||||
## 9. Prompt IDs and Output Artifact Types
|
||||
|
||||
Narratio should treat prompt IDs as configuration, not hardcoded business logic.
|
||||
|
||||
Narratio config may map stage/output names to prompt IDs, for example:
|
||||
|
||||
- session recap prompt
|
||||
- structured event extraction prompt
|
||||
- glossary suggestion prompt
|
||||
- player-facing summary prompt
|
||||
|
||||
Prompt IDs used by Narratio should come from the deployed Scriptorium prompt library.
|
||||
|
||||
## 10. Profiles
|
||||
|
||||
- Prompts may declare `default_profile`.
|
||||
- Narratio may omit `--profile` to use prompt default profile.
|
||||
- Narratio may pass `--profile` to force profile selection.
|
||||
- This enables environment/profile selection like `local-fast`, `local-quality`, `frontier`, `batch`, or test profiles.
|
||||
- Profile names should generally be Narratio configuration values.
|
||||
|
||||
## 11. Runtime Overrides
|
||||
|
||||
Supported runtime override flags:
|
||||
|
||||
- `--llm-base-url`
|
||||
- `--model`
|
||||
- `--api-key-env`
|
||||
- `--temperature`
|
||||
- `--max-tokens`
|
||||
- `--top-p`
|
||||
- `--timeout`
|
||||
|
||||
Guidance:
|
||||
|
||||
- Keep normal model/runtime settings in Execution Profiles.
|
||||
- Use runtime overrides only for explicit per-run exceptions, tests, or operator overrides.
|
||||
- Never pass raw API keys on the command line.
|
||||
- `--api-key-env` names an environment variable; Narratio must ensure that variable is set in subprocess environment.
|
||||
|
||||
## 12. Config Behavior
|
||||
|
||||
- Default config path: `/etc/scriptorium/config.yml`.
|
||||
- `--config <PATH>` overrides default path.
|
||||
- Missing default config is allowed by Scriptorium.
|
||||
- If `--config` is provided explicitly, the file must exist and be valid.
|
||||
- CLI flags override `config.yml`.
|
||||
- `config.yml` overrides built-in application defaults.
|
||||
|
||||
Narratio can either:
|
||||
|
||||
- rely on system default config path, or
|
||||
- carry an explicit config path and pass `--config`.
|
||||
|
||||
## 13. Environment Handling
|
||||
|
||||
Subprocess environment recommendations:
|
||||
|
||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
||||
- Do not pass raw API keys as CLI arguments.
|
||||
- Avoid logging full environment dumps.
|
||||
- Capture stdout and stderr separately.
|
||||
- Use a controlled working directory.
|
||||
- Prefer absolute artifact paths.
|
||||
|
||||
## 14. Output Handling
|
||||
|
||||
For `scriptorium run`:
|
||||
|
||||
- Use `--out` when Narratio needs durable artifact files.
|
||||
- Without `--out`, artifact content is written to stdout.
|
||||
- Preferred orchestration pattern: always use `--out`, then treat the file as stage output artifact.
|
||||
- Capture stderr for diagnostics.
|
||||
|
||||
For `scriptorium render`:
|
||||
|
||||
- Use `--out` to store render diagnostics.
|
||||
- Use `--format json` when tests need to inspect selected profile, effective runtime settings, input hashes, prompt hash, and rendered messages.
|
||||
|
||||
## 15. Exit Status and Errors
|
||||
|
||||
Current CLI behavior (verified from implementation/tests):
|
||||
|
||||
- `0`: success.
|
||||
- `1`: runtime/parse/config/load/render/generation/IO error.
|
||||
- `2`: run completed but output validation failed (`ValidationFailed`).
|
||||
|
||||
Additional details:
|
||||
|
||||
- On `run`, output artifact write happens before exit code selection. If validation fails, artifact may still be written and exit code is `2`.
|
||||
- `stderr` carries both errors and normal run summary output; non-empty stderr alone does not imply failure.
|
||||
- `render` returns `0` on success and `1` on failures.
|
||||
|
||||
Narratio should treat non-zero exit codes as failed stage execution, but may record generated artifact paths if a run exited `2` and output file exists.
|
||||
|
||||
## 16. Recommended Narratio Integration Pattern
|
||||
|
||||
1. Build CLI args from Narratio stage configuration.
|
||||
2. Use subprocess context cancellation/timeout.
|
||||
3. Pass absolute input paths.
|
||||
4. Pass `--out` to a session-scoped artifact path.
|
||||
5. Add `--var` metadata values.
|
||||
6. Optionally add `--config`.
|
||||
7. Optionally add `--profile`.
|
||||
8. Ensure required API-key env vars are present.
|
||||
9. Run subprocess synchronously.
|
||||
10. Capture stdout/stderr separately.
|
||||
11. On success, store output artifact path and invocation metadata in stage artifacts.
|
||||
12. On failure, store exit code and stderr diagnostics in stage status.
|
||||
|
||||
## 17. Suggested Narratio Configuration Shape
|
||||
|
||||
Illustrative `pipeline.yml` shape:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-quality # optional
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
render_debug: false # optional artifact override
|
||||
inputs:
|
||||
transcript:
|
||||
source: trimmed_transcript
|
||||
required: true
|
||||
previous_recap:
|
||||
source: previous_session_artifact
|
||||
artifact: session_recap
|
||||
path: "" # optional
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
```
|
||||
|
||||
The key idea: map Narratio artifact names to prompt ID, optional profile, expected inputs, vars, and output destination.
|
||||
|
||||
## 18. Testing Strategy for Narratio Integration
|
||||
|
||||
- Use `scriptorium render --format json` to verify command construction without LLM calls.
|
||||
- Use dedicated test prompt/profile libraries for integration tests.
|
||||
- Use small fixture transcripts.
|
||||
- Verify missing-input failure behavior.
|
||||
- Verify prompt `default_profile` behavior.
|
||||
- Verify explicit `--profile` override behavior.
|
||||
- Verify `--config` behavior (default and explicit).
|
||||
- Verify output file creation when `--out` is used.
|
||||
- Verify stderr capture on failures.
|
||||
- Avoid real API keys in tests.
|
||||
|
||||
## 19. Security and Privacy Notes
|
||||
|
||||
- Never pass raw API keys on command line.
|
||||
- Do not log full rendered prompts by default; transcripts may contain sensitive content.
|
||||
- Avoid logging prompt content unless explicit debug mode is enabled.
|
||||
- Treat generated artifacts as potentially sensitive.
|
||||
- Use session-scoped, access-controlled output paths.
|
||||
- `api_key_env` names should come from environment management, not embedded secrets.
|
||||
|
||||
## 20. Initial D&D Artifact Generation Examples
|
||||
|
||||
These are examples only. Use prompt IDs from the deployed prompt library.
|
||||
|
||||
Session recap:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.session_recap \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input glossary=/work/session-42/glossary.yml \
|
||||
--out /work/session-42/artifacts/session_recap.md
|
||||
```
|
||||
|
||||
Structured events:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.structured_events \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--out /work/session-42/artifacts/structured_events.json
|
||||
```
|
||||
|
||||
Glossary suggestions:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.glossary_suggestions \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input previous_recap=/work/session-41/artifacts/session_recap.md \
|
||||
--out /work/session-42/artifacts/glossary_suggestions.md
|
||||
```
|
||||
|
||||
Player-facing summary:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.player_summary \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input structured_events=/work/session-42/artifacts/structured_events.json \
|
||||
--out /work/session-42/artifacts/player_summary.md
|
||||
```
|
||||
|
||||
## 21. Non-Goals
|
||||
|
||||
Initial Narratio integration should not:
|
||||
|
||||
- call Scriptorium internal Go packages
|
||||
- use HTTP API as the primary path
|
||||
- expect Scriptorium to read S3 refs directly
|
||||
- make Scriptorium responsible for Narratio stage state
|
||||
- make Scriptorium responsible for notification
|
||||
- require Scriptorium to understand D&D workflow semantics beyond prompt definitions
|
||||
|
||||
## 22. Future Extension Notes
|
||||
|
||||
Possible later extensions:
|
||||
|
||||
- HTTP API integration
|
||||
- S3 artifact references if Scriptorium adds S3 reader support
|
||||
- richer render diagnostics and policy controls
|
||||
- token budgeting/prompt-size checks
|
||||
- batch execution if Scriptorium later adds batch support
|
||||
The request timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
## Request Contract
|
||||
Both request types carry:
|
||||
- binary/config/prompt/profile IDs;
|
||||
- input map and vars map;
|
||||
- output path;
|
||||
- timeout;
|
||||
- generated config + stdout/stderr log paths;
|
||||
- optional API-key env var name;
|
||||
- optional working directory.
|
||||
|
||||
## Result Contract
|
||||
`ArtifactResult` returns:
|
||||
- output/log/generated-config paths;
|
||||
- exit code and duration;
|
||||
- command mode (`run` or `render`);
|
||||
- prompt/profile provenance;
|
||||
- `ValidationFailed` marker;
|
||||
- metadata map.
|
||||
|
||||
## Validation and Failure Semantics
|
||||
Request validation fails for:
|
||||
- missing binary, prompt id, or output path;
|
||||
- non-positive timeout;
|
||||
- empty input/var names;
|
||||
- empty input path values;
|
||||
- missing required credential env var when `APIKeyEnv` is set.
|
||||
|
||||
Run behavior:
|
||||
- subprocess errors propagate with context;
|
||||
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||
- successful subprocess still fails if output file is missing or empty.
|
||||
|
||||
Each artifact result is limited to 64 MiB and must be a regular file without
|
||||
symlinked path components.
|
||||
|
||||
Render behavior:
|
||||
- subprocess errors propagate;
|
||||
- output file must exist and be non-empty.
|
||||
|
||||
## Deterministic Behavior
|
||||
- input and var maps are sorted into deterministic `--input` and `--var` CLI args.
|
||||
- stage wiring adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream routing, overriding any configured `vars.session_id`.
|
||||
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
|
||||
- adapter is stateless and does not own artifact-selection policy.
|
||||
|
||||
## Configuration
|
||||
|
||||
Operator-selected values are defined under `pipeline.scriptorium.*`, including
|
||||
per-artifact settings under `pipeline.scriptorium.artifacts.*`, in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Maintained examples with Scriptorium config:
|
||||
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
@@ -1,403 +1,62 @@
|
||||
# seriatim
|
||||
# Integration: Seriatim
|
||||
|
||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
||||
## Purpose
|
||||
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
|
||||
|
||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
||||
## External Boundary
|
||||
|
||||
## Usage
|
||||
Narratio invokes Seriatim as a subprocess in these modes:
|
||||
|
||||
Run from source:
|
||||
- `seriatim merge`
|
||||
- `seriatim normalize`
|
||||
- `seriatim trim`
|
||||
- `seriatim render`
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file samples/raw/2026-04-19-Eric_Rakestraw.json \
|
||||
--input-file samples/raw/2026-04-19-Mike_Brown.json \
|
||||
--output-file merged.json
|
||||
```
|
||||
The configured timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
Optional report output:
|
||||
## Request/Result Contracts
|
||||
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
|
||||
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
|
||||
- `TrimRequest`/`TrimResult`: transcript trimming with required keep selector.
|
||||
- `RenderRequest`/`RenderResult`: transcript-to-markdown rendering with explicit format and render booleans.
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file eric.json \
|
||||
--input-file mike.json \
|
||||
--output-file merged.json \
|
||||
--report-file report.json
|
||||
```
|
||||
Results include output/log/config paths, timing, exit code, and metadata.
|
||||
|
||||
## CLI
|
||||
## Validation and Failure Semantics
|
||||
Runner construction validates:
|
||||
- binary presence;
|
||||
- timeout > 0;
|
||||
- supported output schema (`seriatim-minimal|seriatim-intermediate|seriatim-full`);
|
||||
- non-negative coalesce gap.
|
||||
|
||||
```text
|
||||
seriatim merge [flags]
|
||||
```
|
||||
Invocation fails on:
|
||||
- missing required request paths/inputs;
|
||||
- invalid normalize schema override;
|
||||
- unsupported render format;
|
||||
- subprocess failure;
|
||||
- invalid JSON outputs for merge/normalize/trim;
|
||||
- missing `segments` array for normalize/trim transcript outputs;
|
||||
- empty render output files.
|
||||
|
||||
Global flags:
|
||||
When report paths are provided/enabled, report files must parse as JSON.
|
||||
Each Seriatim JSON or rendered-text result is limited to 64 MiB and must be a
|
||||
regular file without symlinked path components.
|
||||
|
||||
| Flag | Description |
|
||||
| --- | --- |
|
||||
| `--help` | Show command help. |
|
||||
| `--version` | Show application version. Local builds default to `dev`; release builds inject the release version. |
|
||||
## Deterministic Behavior
|
||||
- argument ordering is deterministic per command construction.
|
||||
- merge env overrides are explicit (`SERIATIM_*`) and only emitted when configured.
|
||||
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
|
||||
- adapter does not write manifests or choose stage inputs.
|
||||
|
||||
`merge` flags:
|
||||
## Configuration
|
||||
|
||||
| Flag | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `--input-file` | Yes | none | Input transcript JSON file. Repeat once per speaker/input file. |
|
||||
| `--output-file` | Yes | none | Merged transcript JSON output path. |
|
||||
| `--report-file` | No | none | Optional report JSON output path. |
|
||||
| `--speakers` | No | none | Speaker map YAML file. When omitted, input file basenames are used as speaker labels. |
|
||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
||||
| `--output-schema` | No | `seriatim-intermediate` | JSON output contract. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. If omitted, the runtime default is used; consumers that depend on a specific shape should set this explicitly. |
|
||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
||||
Operator-selected values are defined under `pipeline.seriatim.*` and
|
||||
`pipeline.render.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Environment variables:
|
||||
Maintained examples with Seriatim config:
|
||||
|
||||
| Environment Variable | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | Output schema used when `--output-schema` is not explicitly provided. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. The CLI flag takes precedence. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
||||
|
||||
## Input JSON Format
|
||||
|
||||
Each input file must be valid JSON with a top-level `segments` array. The current parser accepts the WhisperX segment subset needed for merging:
|
||||
|
||||
```json
|
||||
{
|
||||
"segments": [
|
||||
{
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"words": [
|
||||
{"word": "Hello", "start": 1.25, "end": 1.55, "score": 0.98},
|
||||
{"word": "there.", "start": 1.7, "end": 2.0}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Required segment fields:
|
||||
|
||||
- `start`: number, must be `>= 0`.
|
||||
- `end`: number, must be `>= start`.
|
||||
- `text`: string.
|
||||
|
||||
Optional word fields:
|
||||
|
||||
- `words`: array of word timing objects.
|
||||
- `words[].word`: string.
|
||||
- `words[].start`: optional number, must be `>= 0` when present.
|
||||
- `words[].end`: optional number, must be `>= start` when present with `start`.
|
||||
- `words[].score`: optional number.
|
||||
- `words[].speaker`: optional raw speaker label string.
|
||||
|
||||
Word-level timing is preserved internally for overlap resolution. If a word is missing `start` or `end`, seriatim keeps the word text, emits a warning in the optional report, and does not use that word as a timing anchor. Word timing is not emitted in the final JSON artifact.
|
||||
|
||||
## Speaker Map Format
|
||||
|
||||
`speakers.yml` maps input files to canonical speaker names using ordered substring rules:
|
||||
|
||||
This file is optional. If `--speakers` is omitted, `seriatim` uses each input file basename as the segment speaker label.
|
||||
|
||||
```yaml
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
|
||||
- speaker: "Mike Brown"
|
||||
match:
|
||||
- "Mike_Brown"
|
||||
- "mb"
|
||||
```
|
||||
|
||||
For each `--input-file`, `seriatim` takes the file basename and evaluates the rules in order. The first rule with a matching substring wins, and no later rules are evaluated.
|
||||
|
||||
For example, this input:
|
||||
|
||||
```text
|
||||
samples/raw/2026-04-19-Eric_Rakestraw.json
|
||||
```
|
||||
|
||||
matches this rule because the basename contains `Eric_Rakestraw`:
|
||||
|
||||
```yaml
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
```
|
||||
|
||||
Important details:
|
||||
|
||||
- Matching is against the input file basename, not the full path.
|
||||
- Matching is case-insensitive.
|
||||
- Rules are evaluated from first to last.
|
||||
- Each rule must have a non-empty `speaker`.
|
||||
- Each rule must have at least one non-empty `match` string.
|
||||
- Duplicate speaker names are invalid.
|
||||
- Every input file must match at least one rule or the command fails.
|
||||
|
||||
Deprecated old format:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
eric.json:
|
||||
speaker: "Eric Rakestraw"
|
||||
```
|
||||
|
||||
The old `inputs:` direct mapping format is no longer supported.
|
||||
|
||||
## Output JSON Format
|
||||
|
||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
||||
|
||||
The named schemas are stable public contracts. If a consumer depends on a specific shape, it should request that schema explicitly at runtime. The runtime default selection may change in a future release.
|
||||
|
||||
The `seriatim-intermediate` schema is the current default selection when neither `--output-schema` nor `SERIATIM_OUTPUT_SCHEMA` is set. It stays close to the minimal schema, but adds optional `categories` on each segment:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-intermediate"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there.",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-full` schema uses the full seriatim envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"input_reader": "json-files",
|
||||
"input_files": ["eric.json", "mike.json"],
|
||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "resolve-danglers", "coalesce", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
||||
"output_modules": ["json"]
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"source": "eric.json",
|
||||
"source_segment_index": 0,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"overlap_group_id": 1
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"source": "eric.json",
|
||||
"source_ref": "word-run:1:1:1",
|
||||
"derived_from": ["eric.json#0"],
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 2.0,
|
||||
"end": 2.5,
|
||||
"text": "Resolved word run",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
],
|
||||
"overlap_groups": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 4.0,
|
||||
"segments": ["eric.json#0", "mike.json#0"],
|
||||
"speakers": ["Eric Rakestraw", "Mike Brown"],
|
||||
"class": "unknown",
|
||||
"resolution": "unresolved"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-minimal` schema emits minimal metadata and compact ordered segments:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-minimal"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
||||
|
||||
Intermediate output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
||||
|
||||
Segments are sorted deterministically by:
|
||||
|
||||
```text
|
||||
(start, end, source, source_segment_index/source_ref, speaker)
|
||||
```
|
||||
|
||||
Final segment IDs are assigned after sorting and start at `1`.
|
||||
|
||||
The public Go output contract is available from:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
```
|
||||
|
||||
The same package embeds machine-readable JSON Schemas in `schema/full-output.schema.json`, `schema/intermediate-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
||||
|
||||
## Overlap Detection
|
||||
|
||||
The default postprocessing pipeline detects overlapping segment groups.
|
||||
|
||||
Overlap behavior:
|
||||
|
||||
- A strict timing overlap is required: `next.start < current_group_end`.
|
||||
- Segments that only touch at a boundary are not grouped.
|
||||
- Groups require at least two distinct speakers.
|
||||
- Transitive overlaps are grouped together.
|
||||
- Segments in detected groups receive `overlap_group_id`.
|
||||
- `overlap_groups[].segments` contains stable references in `source#source_segment_index` format.
|
||||
- `class` is currently `unknown`.
|
||||
- `resolution` is `unresolved` until `resolve-overlaps` replaces the group.
|
||||
|
||||
## Overlap Resolution
|
||||
|
||||
The default postprocessing pipeline runs `detect-overlaps`, then `resolve-overlaps`, then `backchannel`, then `filler`, then `resolve-danglers`, then `coalesce`, then a second `detect-overlaps` pass.
|
||||
|
||||
For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word timing to build smaller word-run replacement segments:
|
||||
|
||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
||||
- Once a segment is selected for replacement, all timed words from that segment participate in word-run construction; the window controls segment selection, not per-word clipping.
|
||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
||||
- The default word-run gap is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
||||
- The default word-run reorder window is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` to a positive number of seconds to override the default.
|
||||
- Replacement segment text is built by joining word text with single spaces.
|
||||
- Replacement segments include `source_ref` and `derived_from`.
|
||||
- Replacement segments omit `source_segment_index` because they are derived from one or more original segments.
|
||||
- Resolved overlap groups are removed before the second detection pass.
|
||||
- Replacement segments are left without `overlap_group_id` until the second detection pass annotates any remaining overlap.
|
||||
- If a speaker has no usable word timing in a group, that speaker's original segment is kept.
|
||||
- If no speakers in a group have usable word timing, the original group and annotations remain unchanged.
|
||||
|
||||
## Backchannels
|
||||
|
||||
The default pipeline runs `backchannel` before `coalesce`. It tags short acknowledgement segments with:
|
||||
|
||||
```json
|
||||
"categories": ["backchannel"]
|
||||
```
|
||||
|
||||
Backchannel matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires a matching acknowledgement phrase, no more than three whitespace-delimited words, and duration no greater than `SERIATIM_BACKCHANNEL_MAX_DURATION` seconds. The default maximum duration is `2.0` seconds.
|
||||
|
||||
## Fillers
|
||||
|
||||
The default pipeline runs `filler` after `backchannel` and before `coalesce`. It tags short filler utterances with:
|
||||
|
||||
```json
|
||||
"categories": ["filler"]
|
||||
```
|
||||
|
||||
Filler matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires only filler tokens such as `um`, `uh`, `er`, `erm`, `ah`, `eh`, `hmm`, `mm`, or repeated combinations of those tokens. Matching segments must contain no more than three whitespace-delimited words and have duration no greater than `SERIATIM_FILLER_MAX_DURATION` seconds. The default maximum duration is `1.25` seconds.
|
||||
|
||||
## Dangler Resolution
|
||||
|
||||
The default pipeline runs `resolve-danglers` before `coalesce` and before the second overlap detection pass. It repairs short derived fragments when they share provenance with a nearby segment:
|
||||
|
||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
||||
- Dangling-start fragments have no more than two words.
|
||||
- Matching uses same-speaker segments with any shared `derived_from` value.
|
||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
||||
|
||||
## Coalescing
|
||||
|
||||
The default pipeline runs `coalesce` after `resolve-danglers` and before the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
||||
|
||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
||||
|
||||
Different-speaker backchannel and filler segments do not block coalescing of surrounding same-speaker segments. Same-speaker backchannel and filler segments are merged normally when they are within `--coalesce-gap`. When same-speaker segments are coalesced, any `backchannel` or `filler` category from the merged inputs is dropped from the coalesced segment.
|
||||
|
||||
## Autocorrect
|
||||
|
||||
Autocorrect is included in the default postprocessing pipeline. If `--autocorrect` is omitted, the module leaves transcript text unchanged and records a skip event in the optional report.
|
||||
|
||||
Enable corrections by passing `--autocorrect`:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file input.json \
|
||||
--autocorrect autocorrect.yml \
|
||||
--output-file merged.json
|
||||
```
|
||||
|
||||
`autocorrect.yml` format:
|
||||
|
||||
```yaml
|
||||
autocorrect:
|
||||
- target: "Hrank"
|
||||
match:
|
||||
- "hrank"
|
||||
- "Frank"
|
||||
|
||||
- target: "Mike Brown"
|
||||
match:
|
||||
- "Mike Pat"
|
||||
```
|
||||
|
||||
Matching behavior:
|
||||
|
||||
- Matching is case-sensitive.
|
||||
- Matches apply only to whole tokens, not substrings inside larger words.
|
||||
- Punctuation and whitespace can surround a match.
|
||||
- Multi-word and hyphenated matches are supported.
|
||||
- Duplicate match strings are invalid, including duplicates across separate rules.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
- Only JSON input is supported.
|
||||
- Overlap resolution depends on WhisperX word timing; groups without usable word timing remain unresolved.
|
||||
- Alternate output formats are not implemented yet.
|
||||
|
||||
## Release Builds
|
||||
|
||||
Local builds record version metadata as `dev`. Release builds should inject the release version with `ldflags`:
|
||||
|
||||
```sh
|
||||
go build -ldflags "-X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=v1.0.0" ./cmd/seriatim
|
||||
```
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
92
docs/integrations/whisperx.md
Normal file
92
docs/integrations/whisperx.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# Integration: WhisperX
|
||||
|
||||
## Purpose
|
||||
|
||||
WhisperX transcribes each prepared speaker audio file for Narratio's
|
||||
`transcribe` stage. Narratio uses an HTTP boundary and installs each successful
|
||||
response as that speaker's raw transcript JSON.
|
||||
|
||||
## HTTP Boundary
|
||||
|
||||
Narratio sends an HTTP `POST` to the configured transcription URL using
|
||||
`multipart/form-data` with:
|
||||
|
||||
- `file`: the audio file, retaining its base filename; and
|
||||
- `language`: the configured language string.
|
||||
|
||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||
not currently require a more specific response schema at this boundary.
|
||||
|
||||
The transcription URL must be an absolute `http` or `https` URL. Each
|
||||
application attempt creates one bounded multipart framing and streams the audio
|
||||
from an open file between that framing; it does not buffer the complete audio
|
||||
or spool a multipart copy. The file field precedes the language field.
|
||||
WhisperX response acquisition is capped at 10 MiB.
|
||||
|
||||
The initial request body and any transport-requested replay use the same
|
||||
multipart boundary and framing. Every body opens an independent audio handle
|
||||
and replay starts at the beginning. Narratio closes all bodies created for the
|
||||
attempt on response, failure, timeout, or cancellation, including when a
|
||||
transport abandons an earlier body. Genuine source read/close failures remain
|
||||
request failures; deliberate early closure of an abandoned original does not
|
||||
poison a valid replay.
|
||||
|
||||
## Request And Result Contract
|
||||
|
||||
Each adapter request identifies a speaker, a readable audio file, and the
|
||||
destination for the raw transcript. The HTTP request carries the audio and
|
||||
language; the speaker identifier remains Narratio orchestration metadata.
|
||||
|
||||
On success, Narratio atomically writes the response body to the requested
|
||||
destination. The adapter result reports that logical output together with the
|
||||
attempt count, final HTTP status when available, elapsed duration, and adapter
|
||||
identity metadata. A failed or invalid response is not installed as the
|
||||
transcript output.
|
||||
|
||||
## Retry, Timeout, And Cancellation
|
||||
|
||||
- The configured timeout applies independently to each HTTP attempt.
|
||||
- `retries` means additional attempts after the first.
|
||||
- HTTP `429`, HTTP `5xx`, attempt timeouts, and network errors are retryable.
|
||||
- Other HTTP `4xx` responses and explicit cancellation are not retryable.
|
||||
- Narratio waits the configured retry delay between attempts and aborts that
|
||||
wait when the parent context is canceled.
|
||||
|
||||
Transport replay occurs inside one application attempt and does not consume an
|
||||
entry from the configured retry budget or increment the reported attempt
|
||||
count. The standard HTTP redirect policy remains in effect; body replay can
|
||||
also support body-preserving redirects.
|
||||
|
||||
## Validation And Failure Semantics
|
||||
|
||||
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
||||
a missing language, a non-positive timeout, negative retries, or a negative
|
||||
retry delay. A request fails before transmission when its audio or output path
|
||||
is missing.
|
||||
|
||||
Non-`2xx` status, transport failure, response-size overflow, invalid JSON, or
|
||||
failure to install the output causes the transcription to fail. Errors include
|
||||
attempt context, and the result retains attempts, final status when available,
|
||||
and elapsed duration for diagnostics.
|
||||
|
||||
Replay makes a fresh request body available to the HTTP transport; it does not
|
||||
provide an exactly-once server guarantee and does not prove recovery from every
|
||||
proxy or protocol-level failure. A server may have received work before the
|
||||
client learns that a request failed. WhisperX operations should therefore be
|
||||
safe to retry for the same input when practical.
|
||||
|
||||
## Determinism And Concurrency
|
||||
|
||||
Each audio request has stable multipart field names, and successful bytes are
|
||||
installed atomically. The transcribe stage may process speaker files in
|
||||
parallel, bounded by the configured concurrency. Each validated speaker is
|
||||
checkpointed independently; any speaker failure still fails the aggregate
|
||||
stage, while committed earlier speakers remain reusable on an ordinary retry.
|
||||
|
||||
## Related Canonical Docs
|
||||
|
||||
- [Configuration](../config.md#pipeline) defines the operator-selected
|
||||
WhisperX URL, language, timeouts, retry policy, and concurrency.
|
||||
- [Adapter implementation](../internal/adapters.md) describes internal wiring.
|
||||
- [Transcribe stage](../internal/stage-transcribe.md) describes stage mechanics,
|
||||
durable artifacts, and manifests.
|
||||
121
docs/internal/adapters.md
Normal file
121
docs/internal/adapters.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# Internal: Adapters
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the adapter interfaces and production composition used by application
|
||||
and stage orchestration. Externally observable protocols and formats belong in
|
||||
the [integration contracts](../integrations/).
|
||||
|
||||
## Adapter Boundaries
|
||||
|
||||
Narratio stage logic depends on adapter interfaces, not transport-specific details.
|
||||
|
||||
Primary adapters:
|
||||
|
||||
- `whisperx.Client`
|
||||
- `seriatim.Runner`
|
||||
- `audita.Runner`
|
||||
- `scriptorium.Runner`
|
||||
- `notarius.Runner`
|
||||
- `storage.ObjectStore`
|
||||
- `notify.Sender`
|
||||
|
||||
## Ownership
|
||||
|
||||
Adapters own:
|
||||
|
||||
- HTTP/subprocess/SDK argument and transport details.
|
||||
- Backend-specific request/response mapping.
|
||||
|
||||
Adapters do not own:
|
||||
|
||||
- stage ordering/skip/force logic;
|
||||
- manifest transitions;
|
||||
- canonical path policy.
|
||||
|
||||
## Default Wiring
|
||||
|
||||
The application composes default adapters per invocation when they were not
|
||||
injected through the stage environment, and only when the selected execution
|
||||
plan needs them:
|
||||
|
||||
- WhisperX HTTP client for `transcribe`.
|
||||
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
|
||||
- Audita subprocess runner for `polish`.
|
||||
- Scriptorium subprocess runner for `trim` or `analyze`.
|
||||
- Notarius subprocess runner for `extract` when extraction is enabled.
|
||||
- Noop notifier (`notify.NoopSender`) for `notify`.
|
||||
- Object store only when required by selected stages/config.
|
||||
|
||||
Remote publish locks are loaded only for a selected, enabled publish that
|
||||
uploads a run. Shared session lifecycle setup still applies to every selected
|
||||
range, but an unselected integration is neither initialized nor validated by
|
||||
application-owned composition. Constructor, clock, randomness, remote-lock,
|
||||
and post-publish deletion boundaries are scoped to the invocation; stages still
|
||||
receive their collaborators through the stage environment. Each selected stage
|
||||
retains its own fail-fast configuration and input validation.
|
||||
|
||||
`session plan` is outside production adapter composition. It performs
|
||||
resume validation and models selected transitions against cloned manifest
|
||||
state without constructing or invoking stage-execution adapters. The shared
|
||||
command configuration loader may still use object storage to retrieve a missing
|
||||
remote session file before planning begins.
|
||||
|
||||
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
||||
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
||||
owner of serializing them as repeated `--reference` arguments before `--json`.
|
||||
It also serializes an effective resume request as `--resume` and a nonempty
|
||||
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
|
||||
adapter, resolves force precedence before constructing the request.
|
||||
Its request may also carry one already-resolved opaque ConfigKit secret
|
||||
reference. Only the Notarius adapter reads that value, rejects an unresolved or
|
||||
empty programmatic reference, and maps the exact configured name/value into the
|
||||
shared subprocess override and sensitive-name fields. It never rereads the
|
||||
parent environment. A nil reference preserves the previous invocation.
|
||||
|
||||
Application-owned object-store construction loads configured filesystem
|
||||
secrets before invoking the invocation's storage factory. Production commands
|
||||
use the concrete storage adapter constructor; focused tests can supply an
|
||||
in-memory backend without changing package state.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||
an owned process group on Linux and macOS. Every terminal path disposes of
|
||||
that owned tree before returning. After a natural leader exit, the runner checks
|
||||
for remaining group members and uses bounded graceful then forceful termination.
|
||||
Cancellation, deadlines, and
|
||||
diagnostic limits use the same terminal disposal path without losing their
|
||||
original result classification. Child environments contain only the execution
|
||||
baseline and adapter-specified values; configured credentials are resolved
|
||||
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
|
||||
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
|
||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||
departed leader's descendant. Unsupported platforms reject owned command
|
||||
execution.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Composition: `internal/app/runner_composition.go`,
|
||||
`internal/app/runner_dependencies.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
- `internal/adapters/seriatim/subprocess_test.go`
|
||||
- `internal/adapters/audita/subprocess_test.go`
|
||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||
- `internal/adapters/notarius/subprocess_test.go`
|
||||
- `internal/adapters/storage/*_test.go`
|
||||
- `internal/app/runner_test.go`
|
||||
|
||||
See the [WhisperX](../integrations/whisperx.md),
|
||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||
[Scriptorium](../integrations/scriptorium.md), and
|
||||
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||
externally visible boundary. Operator-selected values belong in
|
||||
[Configuration](../config.md).
|
||||
302
docs/internal/artifacts.md
Normal file
302
docs/internal/artifacts.md
Normal file
@@ -0,0 +1,302 @@
|
||||
# Internal: Artifacts
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the artifact registry, runtime catalog, resolver, previous-input
|
||||
requirements, and shared remote current-state mechanics implemented by
|
||||
`internal/artifacts`. Configuration fields that accept source IDs belong in
|
||||
[Configuration](../config.md); physical placement belongs in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Built-in Source IDs
|
||||
|
||||
The internal registry recognizes these stable built-in source IDs:
|
||||
|
||||
- `narratio.transcript.base`
|
||||
- `narratio.transcript.polished`
|
||||
- `narratio.transcript.final`
|
||||
- `narratio.transcript.final_trimmed`
|
||||
- `narratio.transcript.final_markdown`
|
||||
- `narratio.transcript.final_trimmed_markdown`
|
||||
- `narratio.bounds.session`
|
||||
|
||||
Registry entries bind each ID to its producer, output kind, and content
|
||||
validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
|
||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
All formats are validated by strict source-policy rules. Configured artifact and
|
||||
extraction keys use `^[a-z][a-z0-9_]*$`; source parsers never normalize an
|
||||
unrecognized token into a valid source. Extraction sources are registered only
|
||||
from `pipeline.notarius.outputs`; the Notarius index has no selectable source
|
||||
ID.
|
||||
|
||||
Prepared stable source IDs are `narratio.input.players`,
|
||||
`narratio.input.party`, `narratio.input.glossary`, and
|
||||
`narratio.input.spell_catalog`. Artifact policy owns their canonical manifest
|
||||
kind and prepared filename vocabulary. Canonical party mode preserves the
|
||||
party source bytes in the party record and supplies the players record from the
|
||||
deterministic `derived_from_party` projection; both remain ordinary prepared
|
||||
source IDs for consumers.
|
||||
|
||||
## Runtime Catalog
|
||||
|
||||
`ArtifactCatalog` tracks:
|
||||
|
||||
- `planned`: source registered for run context;
|
||||
- `executable`: included in the effective analyze artifact set;
|
||||
- `available`: the source's canonical evidence owner validates its current
|
||||
manifest record and durable bytes;
|
||||
- `provenance`: availability source.
|
||||
|
||||
Configured definitions are always registered. Without an explicit selection,
|
||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||
exact named configured definitions become the effective set for that invocation,
|
||||
regardless of their `enabled` value. The effective-set resolver itself does not
|
||||
expand dependencies. Configuration has already materialized the sorted union of
|
||||
explicit ordering edges and configured-artifact input edges, and the analyze
|
||||
work planner closes targets over that one concrete prerequisite graph.
|
||||
Availability is separate from executability. Configuration may normalize a
|
||||
family selection into its concrete generated members before this resolver runs.
|
||||
The effective set retains optional family
|
||||
and character origin metadata, but its keys, catalog sources, and runtime
|
||||
lookups remain concrete configured-artifact identities.
|
||||
Family publish policies are likewise expanded into ordinary configured-source
|
||||
publish rules during configuration resolution.
|
||||
Configured outputs, including non-executable prerequisites, become available
|
||||
only when the versioned analyze state identifies a current result whose source,
|
||||
contract, canonical configured path, size, and checksum match a confined
|
||||
no-follow regular file. An incidental canonical file is unavailable, and a
|
||||
manifest with an unsupported aggregate-only representation is rejected before
|
||||
catalog construction.
|
||||
Extraction entries are registered from configuration and become available only
|
||||
after compatible extraction evidence is hydrated.
|
||||
|
||||
During an analyze invocation, a newly validated and atomically materialized
|
||||
configured output is marked available with its producer run ID, contract,
|
||||
checksum, and size. Later scheduled dependents therefore observe the same
|
||||
semantic identity whether their prerequisite was reused from current manifest
|
||||
evidence or produced earlier in the invocation.
|
||||
|
||||
Current provenance values:
|
||||
|
||||
- `generated.current_analyze_run`
|
||||
- `manifest.current_analyze_artifact`
|
||||
- `manifest.inputs.previous_cache`
|
||||
- `current_session.previous_cache`
|
||||
|
||||
## Resolution Rules
|
||||
|
||||
Built-ins:
|
||||
|
||||
- resolve only from the current manifest's succeeded registered producer;
|
||||
- require exactly one output with the registered source ID and output kind, a
|
||||
safe session-rooted path, producer run ID, canonical checksum, and valid
|
||||
content; and
|
||||
- verify the registered checksum against the durable bytes. Incidental files,
|
||||
canonical filenames, duplicate outputs, failed/stale producer records, and
|
||||
malformed evidence are unavailable.
|
||||
|
||||
Configured sources (`narratio.artifact.*`):
|
||||
|
||||
- resolve only through runtime catalog availability;
|
||||
- use the shared typed analyze-evidence inspection in
|
||||
`analyze_evidence.go` for prior current-session results;
|
||||
- require the supported analyze-state and fingerprint versions, a `current`
|
||||
record for the exact configured key and source ID, a complete contract, the
|
||||
configured canonical relative path, positive stored size, and stored
|
||||
checksum matching bytes read from a confined no-follow regular file; and
|
||||
- treat non-current statuses, malformed records, removed keys,
|
||||
unsafe or missing files, and size/checksum mismatches as unavailable without
|
||||
rewriting manifest state. Catalog construction iterates current
|
||||
configuration, so removed or renamed records are not advertised.
|
||||
|
||||
`narratio.member_artifact.*` is not a runtime source family. Configuration
|
||||
resolution accepts it only in an artifact-family declaration and rewrites it
|
||||
to the corresponding configured source before this catalog is built.
|
||||
|
||||
Prepared stable sources (`narratio.input.*`):
|
||||
|
||||
- resolve only from the current manifest's exact prepared-input record;
|
||||
- require the policy-owned canonical path below the session root, a confined
|
||||
non-symlink regular file, a non-empty payload, and a matching SHA-256
|
||||
checksum; and
|
||||
- return an immutable source/path/checksum/size identity shared by extract and
|
||||
analyze rather than falling back to campaign/session source paths.
|
||||
|
||||
Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
|
||||
- require a current successful extract record with the exact configured source,
|
||||
compatible contract and Notarius provenance, a confined regular durable
|
||||
payload, matching checksum, and the current resolved trimmed-transcript
|
||||
identity;
|
||||
- remain unavailable unless catalog hydration receives valid evidence. Resume
|
||||
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
|
||||
an error; and
|
||||
- are never inferred by scanning the Notarius bundle directory.
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from one matching `previous_artifact` input record in the
|
||||
current manifest, including its exact source ID and checksum;
|
||||
- are hydrated only from the unique `published_output` declaration for their
|
||||
canonical source in the selected remote commit; and
|
||||
- never infer a path from a previous manifest, current configuration, basename,
|
||||
or existing cache file.
|
||||
|
||||
Source absence is evaluated by the consuming artifact input. An optional input
|
||||
is omitted from that invocation; a required input fails resolution. This is
|
||||
separate from a stage's lifecycle outcome.
|
||||
|
||||
Validation by content type:
|
||||
|
||||
- transcript JSON built-ins: JSON with top-level `segments` array;
|
||||
- transcript Markdown built-ins: non-empty text file;
|
||||
- bounds built-in: valid JSON;
|
||||
- configured/previous-session artifact files: non-empty text file.
|
||||
|
||||
## Previous Requirement Collection
|
||||
|
||||
`CollectPreviousArtifactRequirements`:
|
||||
|
||||
- scans the effective configured artifact set;
|
||||
- extracts only canonical previous-session sources;
|
||||
- deduplicates by artifact key;
|
||||
- merges required and optional references (required wins);
|
||||
- returns deterministic ordering and source locations.
|
||||
|
||||
## Current-State Helpers
|
||||
|
||||
Artifacts package owns shared remote current-state loading mechanics used by
|
||||
restore, status and validation checks, and previous-cache planning.
|
||||
|
||||
The selected session manifest is accepted only with
|
||||
`schema_version: narratio.session-manifest.v1` and complete stage identity.
|
||||
An unsupported local or downloaded manifest must be replaced by regeneration,
|
||||
verified restore, or republication; artifact resolution never fills in missing
|
||||
authority markers.
|
||||
|
||||
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||
complete restore authority. Callers receive its declared object identities and
|
||||
must not supplement them by listing mutable session prefixes.
|
||||
|
||||
A `published_output` declaration must use a canonical publish-compatible source
|
||||
ID and an exact `runs/{run_id}/published/{destination}` key in the same session.
|
||||
Only this declaration type may refer to an earlier run; other declared objects
|
||||
remain confined to the selected commit's run.
|
||||
|
||||
The reader opens each small control object directly and enforces owner-specific
|
||||
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. These
|
||||
are exposed as `MaxCurrentCommitPointerBytes`,
|
||||
`MaxRemoteCommitManifestBytes`, and `MaxRemoteSessionManifestBytes`.
|
||||
|
||||
Each read uses the generation and size metadata returned with its opened body.
|
||||
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||
checksum, generation, and identity checks. No current-state control object is
|
||||
downloaded through a temporary file. Artifact and previous-session downloads
|
||||
validate their complete sibling temporary file against available committed
|
||||
identity before replacing a canonical local path.
|
||||
|
||||
Core helpers:
|
||||
|
||||
- `LoadCurrentState`
|
||||
- `ValidateCurrentStateIdentity`
|
||||
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||
|
||||
Typed missing-state error:
|
||||
|
||||
- `CurrentCommitPointerMissingError` (`ErrCurrentCommitPointerMissing`)
|
||||
|
||||
Identity validation supports caller-provided expectations:
|
||||
|
||||
- expected campaign;
|
||||
- expected session ID;
|
||||
- expected run ID, or pointer/manifest run-ID consistency check.
|
||||
|
||||
Caller policy is intentionally outside artifacts helpers:
|
||||
|
||||
- some callers fail on missing current state;
|
||||
- some callers downgrade missing state to status/findings;
|
||||
- some callers skip optional behavior when state is missing.
|
||||
|
||||
## Remote Lock Objects
|
||||
|
||||
The artifacts package defines the independent remote-lock payload and identity
|
||||
contract. Each canonical publish source maps to one object below the validated
|
||||
session `locks/` prefix; its filename is the lowercase SHA-256 digest of the
|
||||
exact source followed by `.lock`.
|
||||
|
||||
Lock payloads are strict versioned JSON containing the canonical source, an
|
||||
optional reason, and a UTC creation time. Reads are bounded to
|
||||
`MaxRemoteLockObjectBytes` (64 KiB), reject unknown fields and trailing values,
|
||||
and verify that the payload source matches the key digest. Exact lookup reports
|
||||
presence explicitly, prefix listing validates every object and sorts by source,
|
||||
creation uses an absence precondition, forced replacement uses one observed
|
||||
generation, and removal is idempotent.
|
||||
|
||||
## Key Path Helpers
|
||||
|
||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||
|
||||
- session/work/run paths;
|
||||
- previous-cache paths;
|
||||
- spool/cache paths;
|
||||
- S3 session/run/current-state key layout.
|
||||
|
||||
New publication creates run-scoped immutable objects, including
|
||||
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||
commit and declared object generations/checksums.
|
||||
|
||||
A `published_output` declaration may identify its commit's run or an earlier
|
||||
immutable run in the same campaign/session namespace. The key parser validates
|
||||
that namespace, rejects mutable and malformed keys, and derives the originating
|
||||
run from the exact declaration. All other commit artifacts remain confined to
|
||||
the commit's own run prefix.
|
||||
|
||||
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||
segments at configuration and artifact boundaries before they can be used in a
|
||||
workspace or S3 namespace. Previous-artifact destinations remain typed,
|
||||
multi-segment relative paths and are confined beneath `previous/artifacts`; they
|
||||
are not treated as opaque identifiers.
|
||||
|
||||
See [Workspace Internals](workspace.md) for how callers consume local helpers
|
||||
and [Operations](../operations.md#local-state-layout) for the authoritative
|
||||
physical layout.
|
||||
|
||||
## Invariants
|
||||
|
||||
- source ID formats are stable contracts;
|
||||
- artifact resolution is deterministic and manifest-aware;
|
||||
- extraction sources are available only from a compatible successful manifest
|
||||
record;
|
||||
- previous-session source resolution in `analyze` is local-only;
|
||||
- remote current-state key construction remains centralized in artifacts helpers.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||
`internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_evidence.go`,
|
||||
`internal/artifacts/extraction_input.go`,
|
||||
`internal/artifacts/prepared_input.go`
|
||||
- Current state: `internal/artifacts/current_state.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||
`internal/artifacts/catalog_test.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`,
|
||||
`internal/artifacts/current_state_test.go`,
|
||||
`internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/previous_requirements_test.go`
|
||||
127
docs/internal/command-restore.md
Normal file
127
docs/internal/command-restore.md
Normal file
@@ -0,0 +1,127 @@
|
||||
# Internal: Command Restore
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the implemented restore discovery, planning, installation, and
|
||||
reporting flow in `internal/app`. User invocation belongs in
|
||||
[CLI](../cli.md#session-restore), and the operator recovery procedure and
|
||||
physical restore scope belong in
|
||||
[Operations](../operations.md#restore-workflow).
|
||||
|
||||
Restore separates remote authority, local conflict policy, and filesystem
|
||||
mutation so each remains testable independently.
|
||||
|
||||
The public command supplies production defaults to one private,
|
||||
invocation-scoped dependency value. Discovery, planning, and execution use that
|
||||
same value, so focused sequencing tests can replace one boundary without
|
||||
mutable package hooks.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Discovery delegates current-state pointer and manifest loading to
|
||||
`internal/artifacts`, then validates the result against the resolved request:
|
||||
|
||||
- campaign must match;
|
||||
- session ID must match.
|
||||
- run ID must match the pointer-selected committed run.
|
||||
|
||||
Restore treats any missing or invalid remote current state as a command error.
|
||||
|
||||
## Planning Contract
|
||||
|
||||
Restore planner action kinds:
|
||||
|
||||
- `download`;
|
||||
- `skip_same`;
|
||||
- `conflict`.
|
||||
|
||||
Planner behavior:
|
||||
|
||||
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||
each action carries that artifact's immutable key, checksum, size, and
|
||||
generation;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted by local relative path and then remote key;
|
||||
- planning opens the selected session root and inspects every target by its
|
||||
normalized session-relative path. Missing ancestors produce a missing target;
|
||||
an ancestor or leaf symlink is a non-regular conflict and is never opened for
|
||||
hashing;
|
||||
- force converts differing eligible regular files from conflicts to downloads;
|
||||
directories, symlinks, and other non-regular targets remain conflicts.
|
||||
|
||||
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||
|
||||
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||
prepare, status, and validation. Each requirement selects the unique
|
||||
`published_output` for its canonical source in the pointer-selected commit,
|
||||
then strips only that run's immutable `published/` prefix for its safe local
|
||||
cache path. It never probes candidate objects or derives paths from manifests,
|
||||
configuration, or basenames.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
Execution order and safety:
|
||||
|
||||
- non-manifest downloads happen before manifest install;
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files, validate the complete object, then use an
|
||||
atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- each committed object is verified against its declared checksum, size, and
|
||||
generation before installation;
|
||||
- installation reopens the selected session root, creates only validated
|
||||
missing directory components with workspace modes, and rejects symlinked or
|
||||
non-regular ancestors and leaves before replacement;
|
||||
- a committed manifest already verified during discovery is retained for the
|
||||
matching restore action and revalidated before installation, avoiding a
|
||||
second body transfer;
|
||||
- failed installs do not roll back files already written in the same execution.
|
||||
- a durable `.restore-incomplete.json` marker is written before installation.
|
||||
It blocks runners until a restore retry completes all verified installs and
|
||||
the local manifest replacement, at which point it is removed.
|
||||
- restored manifest local references are rebased beneath the selected local
|
||||
session root. Unsafe relative references and producer-machine absolute paths
|
||||
outside the manifest's producer session root are rejected; producer-local
|
||||
spool paths and run-local diagnostic references are not restored as
|
||||
authority. Cleanup remains ordinary restored stage state and is not reset or
|
||||
synthesized by restore.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- reuses cached audio only when its opened regular file, content digest, and
|
||||
identity sidecar all match the selected remote object version; otherwise it
|
||||
refreshes through the validated download path.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
- dry-run mode prints a summary, performs no durable session writes, and may
|
||||
read remote current-state or object-identity data to produce that summary;
|
||||
- execution mode persists the canonical restore report described in
|
||||
[Operations](../operations.md#restore-workflow);
|
||||
- report includes plan counts, per-action status, and execution failures.
|
||||
|
||||
## Invariants
|
||||
|
||||
- restore uses committed remote current state as authority;
|
||||
- one restore or status inspection observes the single pointer-selected commit
|
||||
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||
different run into its plan;
|
||||
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||
establish remote commitment; when absent, republish the session;
|
||||
- restore does not execute pipeline stages.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Discovery: `internal/app/restore_discovery.go`
|
||||
- Planning: `internal/app/restore_plan.go`, `internal/previouscache`
|
||||
- Execution: `internal/app/restore_execute.go`
|
||||
- Reporting and command coordination: `internal/app/restore_report.go`,
|
||||
`internal/app/restore.go`
|
||||
- Tests: `internal/app/restore_discovery_test.go`,
|
||||
`internal/app/restore_plan_test.go`,
|
||||
`internal/app/restore_execution_test.go`,
|
||||
`internal/app/restore_recovery_test.go`,
|
||||
`internal/app/restore_workflow_test.go`
|
||||
215
docs/internal/configuration.md
Normal file
215
docs/internal/configuration.md
Normal file
@@ -0,0 +1,215 @@
|
||||
# Configuration Internals
|
||||
|
||||
User-visible fields, defaults, and selection behavior belong in the
|
||||
[Configuration Reference](../config.md). This document describes the internal
|
||||
pipeline-loading boundary implemented by `internal/config`.
|
||||
|
||||
## Pipeline Loading
|
||||
|
||||
`LoadPipeline` assembles and validates a pipeline in this order:
|
||||
|
||||
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
|
||||
enabled imports, and the caller's profile selection. Omitted, empty, and
|
||||
exact `default` selections use the base; named profiles use the fixed
|
||||
`profiles/<name>.yml` convention.
|
||||
2. ConfigKit parses the root and declared imports, additively merges base
|
||||
sources, applies only the selected overlay, and strictly decodes the result.
|
||||
3. The definition resolver applies recorded entry-level and inherited defaults,
|
||||
runs deterministic domain normalization, records derived publish destinations,
|
||||
and resolves ordinary paths relative to the root pipeline file.
|
||||
4. The definition validator performs read-only checks that require only the
|
||||
pipeline file. Checks requiring canonical-party family expansion remain at
|
||||
the existing explicit pipeline/campaign resolution boundary.
|
||||
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
|
||||
declared by multiple base files, adapts source ownership records, attaches
|
||||
runtime-only state, and computes the normalized effective digest.
|
||||
|
||||
This ordering preserves monolithic configuration behavior. Moving a field to
|
||||
an imported fragment changes its source ownership, not its path base, default,
|
||||
or schema semantics.
|
||||
|
||||
## Loaded Context Resolution
|
||||
|
||||
`LoadedPipelineCampaign` carries one already composed pipeline and its selected
|
||||
campaign into session resolution. `LoadSessionWithPipelineCampaignOptions`
|
||||
loads a local session against that context, while
|
||||
`ResolveLoadedPipelineCampaign` also accepts an already loaded remote session
|
||||
or no session while a caller retrieves one. Compatibility loaders route through
|
||||
these functions after their initial pipeline and campaign reads.
|
||||
|
||||
Application commands own pipeline and campaign discovery, campaign-file versus
|
||||
registry selection, and the corresponding mutual-exclusion rules. Once they
|
||||
have a `LoadedPipelineCampaign`, local session discovery and remote-session
|
||||
download retain that exact pipeline object and its private provenance. Removing
|
||||
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||
pipeline or campaign context.
|
||||
|
||||
Campaign and session files use independent ConfigKit definitions with imports
|
||||
and profiles disabled. Campaign files normally load directly from their source;
|
||||
session files are first read through the 8 MiB bounded-read boundary because the
|
||||
application must reject unresolved template placeholders before passing an exact
|
||||
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
|
||||
the same typed loader and retain the existing post-load identity expectations,
|
||||
stable-input precedence, and source-relative path semantics.
|
||||
|
||||
The application also has a separate read-only inspection resolver for `config
|
||||
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
|
||||
campaign selection functions, but never routes through session discovery,
|
||||
remote-session download, adapter composition, workspace
|
||||
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
||||
family declarations must resolve its selected campaign before ordinary pipeline
|
||||
validation, which expands its canonical-party members and generated publish
|
||||
rules. A pipeline without those declarations may be validated by itself.
|
||||
Secret-directory bootstrap remains invocation-scoped and happens before the
|
||||
ordinary ConfigKit load because explicit typed references must resolve even for
|
||||
inspection.
|
||||
|
||||
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
||||
show`. It serializes one explicit typed safe projection with the existing YAML
|
||||
encoder. The projection has no slots for resolution-only artifact family
|
||||
declarations or private provenance. The result has one trailing newline;
|
||||
commands do not marshal runtime objects directly.
|
||||
|
||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||
the complete logical field paths projected from the schema and classifies
|
||||
each contributor as root, import, profile, or centralized default. The
|
||||
projection replaces generated concrete member paths with paired family and
|
||||
canonical-party records, and does the same for generated publish rules.
|
||||
Campaign records identify campaign-owned fields and party inputs; canonical
|
||||
derived players point to the party source. The application command only joins these sorted
|
||||
records with selection metadata and never reparses configuration files.
|
||||
|
||||
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||
family/character/party provenance for generated members. It does not inspect
|
||||
files named after artifacts or infer another runtime graph.
|
||||
|
||||
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||
loads. When campaign
|
||||
resolution is needed, the command loads one selected campaign and party and
|
||||
expands both effective pipelines from that same party value. The configuration
|
||||
owner projects each normalized effective mapping directly from that same typed
|
||||
projection into sorted logical paths using JSON number preservation;
|
||||
mapping leaves are compared individually while sequence values remain atomic.
|
||||
Values are compact deterministic JSON representations for command output, not
|
||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||
with no projected difference is treated as an internal consistency error.
|
||||
|
||||
Campaign context construction reads the campaign-owned party source through
|
||||
`ParseParty`. The required canonical party retains its raw bytes and normalized
|
||||
roster in runtime-only `ResolvedParty` provenance. Canonical resolution creates
|
||||
a deterministic `derived_from_party` players projection. The party is the only
|
||||
configured roster authority; player and session-party fields are unsupported.
|
||||
|
||||
## Canonical Party Domain
|
||||
|
||||
`ParseParty` is the package-owned boundary for parsing a party source. Its
|
||||
single-source ConfigKit definition disables imports and profiles, requires
|
||||
`schema_version` through `VersionRule`, and strictly decodes the
|
||||
character-keyed typed schema before applying Narratio's party-domain checks.
|
||||
Successful metadata coordinates recover authored character-map order without a
|
||||
second YAML parse; class and alias slice order comes directly from typed decode.
|
||||
The canonical value retains a separate exact byte copy of its source so
|
||||
consumers can materialize the authored party document without reserializing it.
|
||||
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
|
||||
players-only projection.
|
||||
|
||||
An unversioned source is rejected before resolution, family expansion, or
|
||||
prepare-stage materialization.
|
||||
|
||||
## Validation Ownership
|
||||
|
||||
Configuration validation remains within `internal/config`, with the public
|
||||
entry points and pipeline-domain sequencing separated from the private owners
|
||||
for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
||||
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||
boundaries rather than extension points or a general validation framework.
|
||||
|
||||
Validation is read-only, fails fast, and does not define which of several
|
||||
independent invalid fields is reported first. Map-backed declarations are
|
||||
nevertheless traversed deterministically. Reusable domain normalization is an
|
||||
explicit preceding operation for both loaded and programmatically constructed
|
||||
pipelines. Notarius normalization commits canonical scalar, reference, and output
|
||||
values only after the complete normalization succeeds; its validator then checks
|
||||
that candidate without changing it. Scriptorium validates artifact declarations
|
||||
and outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||
dependency graph in deterministic artifact order.
|
||||
|
||||
Diagnostics identify the relevant semantic field. Merge conflicts also
|
||||
retain every claiming source needed to correct an additive ownership conflict.
|
||||
Complete prose, punctuation, formatting, and private validation choreography are
|
||||
not compatibility contracts.
|
||||
|
||||
## Diagnostics And Runtime Metadata
|
||||
|
||||
Syntax, duplicate-key, merge, profile, and schema failures include the
|
||||
relevant source name and full field path. Additive conflicts report every
|
||||
claiming source so operators can repair the split without repeatedly
|
||||
rediscovering additional conflicts.
|
||||
|
||||
The loaded pipeline retains private runtime metadata for the absolute root
|
||||
path, ordered imports, selected profile name and selection source, selected
|
||||
overlay, original ConfigKit metadata and source snapshots, effective digest,
|
||||
and leaf ownership. Base
|
||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
||||
metadata does not participate in YAML decoding or alter the public
|
||||
configuration model.
|
||||
|
||||
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
|
||||
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
|
||||
execution, but the digest substitutes their normalized logical values captured
|
||||
before root-relative resolution, so relocating an equivalent configuration
|
||||
bundle does not change provenance. Because ConfigKit and resolution metadata
|
||||
are private, the digest excludes source layout, profile name, and ownership.
|
||||
Configuration stores credentials as ConfigKit typed references. A successful
|
||||
load resolves their values inside the opaque typed boundary, while this safe
|
||||
projection serializes only `EnvName()` identities. Credential-value changes are
|
||||
therefore never hashed; reference-name changes remain ordinary provenance.
|
||||
The optional Notarius credential follows the same path even for disabled or
|
||||
resumable extraction; normalization copies the opaque reference without reading
|
||||
or resolving it again. Direct YAML serialization has a configuration-owned safe
|
||||
projection so only the reference name can leave this boundary.
|
||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||
for later runtime expansion.
|
||||
|
||||
## Test Surfaces
|
||||
|
||||
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
|
||||
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
||||
unique artifact/family base-declaration rule. Existing focused configuration
|
||||
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||
single-read boundary by changing the pipeline file after its initial load and
|
||||
confirming local session resolution retains the original pipeline. Other
|
||||
configuration tests continue to protect defaults and validation after assembly.
|
||||
`party_test.go` protects the versioned typed party schema, domain invariants,
|
||||
authored character order, retained bytes, and deterministic players projection
|
||||
without involving campaign or runtime wiring.
|
||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||
input restrictions, source provenance, and virtual players input selection.
|
||||
|
||||
## Artifact Family Resolution
|
||||
|
||||
Pipeline loading retains `scriptorium.artifact_families` as a resolution-only
|
||||
declaration. Once campaign party resolution establishes a canonical roster,
|
||||
configuration expands families in sorted family-key and character-ID order
|
||||
into ordinary `ScriptoriumArtifactConfig` values. The expansion owns the narrow
|
||||
`{character_id}` output substitution, closed member-variable selectors, key and
|
||||
output collision checks, and the runtime-only family-origin catalog. It then
|
||||
removes family declarations from `ScriptoriumConfig`, runs ordinary Scriptorium
|
||||
validation, and refreshes the effective pipeline digest. Stages and adapters
|
||||
therefore receive only concrete artifact maps.
|
||||
|
||||
The catalog retains sorted family member keys plus family/character/source
|
||||
origins and the typed dependency/publish declarations for their later owners.
|
||||
`member_dependencies` add corresponding ordinary concrete dependencies, while
|
||||
the family-only `narratio.member_artifact.<family>` input form is rewritten to
|
||||
the matching ordinary configured-artifact source. The catalog records those
|
||||
resolved dependency and input identities with their declaring family and party
|
||||
member. No member-artifact source is registered as a runtime policy source.
|
||||
An enabled family publish declaration expands to ordinary configured-artifact
|
||||
publish rules before the existing publish and lock validators run. Runtime
|
||||
publication consequently receives no family wildcard or special matcher.
|
||||
55
docs/internal/fileops.md
Normal file
55
docs/internal/fileops.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Internal: File Operations
|
||||
|
||||
`internal/fileops` owns the narrow mechanics for atomically visible replacement of one
|
||||
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||
and destination-directory policy.
|
||||
|
||||
## Destination Paths
|
||||
|
||||
Ordinary workspace replacement uses standard operating-system path semantics.
|
||||
Callers with a root-scoped safety contract own that validation at their boundary.
|
||||
|
||||
Remote object acquisition is owned by its domain caller. The caller writes a
|
||||
sibling temporary file, validates the complete result, and passes that file to
|
||||
`InstallSiblingTempFile` only when it is ready to become canonical. Callers own
|
||||
remote object selection, validation, conflict handling, and temporary-file
|
||||
cleanup.
|
||||
|
||||
Directory bundles are not promoted by `internal/fileops`; extraction owns its
|
||||
validated Notarius bundle rename.
|
||||
|
||||
## Cleanup Contract
|
||||
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant, opens
|
||||
that root with `os.OpenRoot`, and removes the normalized root-relative target
|
||||
with `Root.RemoveAll`. It rejects root, traversal, and out-of-root targets; a
|
||||
missing target is successful. `RemoveFileUnderRoot` additionally requires an
|
||||
existing leaf to be regular. Command and post-publish policy remains owned by
|
||||
`internal/app`.
|
||||
|
||||
## Bounded Reads
|
||||
|
||||
`ReadFileBounded` opens a normal path and reads at most its caller-supplied
|
||||
limit. Credential mode policy and environment precedence remain owned by
|
||||
`internal/app`.
|
||||
|
||||
## Replacement Contract
|
||||
|
||||
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||
sibling temporary file, writes the complete byte sequence, applies the
|
||||
caller-supplied mode, closes the file, runs an optional pre-rename check, and
|
||||
replaces the destination with a rename.
|
||||
|
||||
The pre-rename check is the last point at which a caller can cancel without
|
||||
installing a new destination. A failure before the rename leaves the old
|
||||
destination unchanged and removes the temporary file. Successful replacement
|
||||
provides atomic visibility; it makes no crash- or power-loss durability claim.
|
||||
|
||||
Replacement follows the operating system's same-filesystem rename semantics.
|
||||
If a platform cannot replace an existing destination, the operation returns an
|
||||
error and never removes the old file as an emulation step.
|
||||
|
||||
`WriteFileAtomic` and the copy helpers create the destination parent with the
|
||||
repository's workspace permissions. `InstallSiblingTempFile` requires the
|
||||
caller to provide an existing temporary file in the existing destination
|
||||
directory.
|
||||
374
docs/internal/manifest.md
Normal file
374
docs/internal/manifest.md
Normal file
@@ -0,0 +1,374 @@
|
||||
# Internal: Manifest
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the session-progress model implemented by `internal/manifest`. Physical
|
||||
manifest placement belongs in
|
||||
[Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Session Manifest
|
||||
|
||||
`manifest.Manifest` records:
|
||||
|
||||
- required `schema_version: narratio.session-manifest.v1`
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map, including the ordinary cleanup stage record
|
||||
|
||||
Session, campaign, and run identities in local and downloaded manifests must be
|
||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||
guidance rather than being normalized into a different workspace or remote
|
||||
namespace.
|
||||
|
||||
Decoding is strict at every typed manifest field. Unknown fields are rejected;
|
||||
the schema version remains `narratio.session-manifest.v1` and there is no
|
||||
compatibility reader that silently drops retired state. Cleanup progress uses
|
||||
only the cleanup entry in `stages`.
|
||||
|
||||
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
||||
non-nil and its embedded `name` exactly matches its map key; loading never
|
||||
invents either marker. A missing or unsupported schema version, or incomplete
|
||||
stage identity, is unsupported local state: move it aside and regenerate it,
|
||||
or restore a verified current committed snapshot when that is authoritative.
|
||||
|
||||
Prepare records independent `party` and `players` input checksums. In canonical
|
||||
party mode, the party record retains its campaign source identity while the
|
||||
players record uses `derived_from_party`; raw roster content is never embedded
|
||||
in manifest metadata. Both records remain the durable authority for consumers
|
||||
of their prepared input source IDs.
|
||||
|
||||
The model admits these stage states:
|
||||
|
||||
- `pending`
|
||||
- `running`
|
||||
- `succeeded`
|
||||
- `failed`
|
||||
- `skipped`
|
||||
- `stale`
|
||||
- `interrupted`
|
||||
|
||||
### Transcribe-owned speaker state
|
||||
|
||||
A `transcribe` stage record may carry `transcribe_state_version: 1` and a
|
||||
`transcribe_speakers` map keyed by the existing derived speaker ID. The version
|
||||
is the authority marker: a versioned empty map means that no earlier speaker
|
||||
completion is reusable. Absence of both fields is accepted as legacy
|
||||
aggregate-only state so current Narratio can read older manifests.
|
||||
|
||||
Each map entry represents one completed speaker only. It binds a versioned
|
||||
fingerprint to the session-relative audio path, audio SHA-256 and size, one
|
||||
canonical `transcript_raw` output with its path, SHA-256 and size, and the
|
||||
producer run ID. Paths must be canonical and session-relative, checksums must
|
||||
be lowercase SHA-256 values, and the record and output producer identities
|
||||
must agree. Other stages cannot carry this transcribe-owned state.
|
||||
Audio size is nonnegative; WhisperX determines whether that audio can be
|
||||
transcribed. Output size remains positive and the transcript must be valid JSON.
|
||||
|
||||
Transcribe reconciles this collection against the current prepared audio set
|
||||
and observable WhisperX semantics. Missing or corrupt regular outputs become
|
||||
work; unsafe or non-regular evidence is rejected. It saves the reconciled
|
||||
collection before HTTP dispatch and saves a detached candidate after each
|
||||
validated completion. Only after a save succeeds is the candidate installed as
|
||||
live authority. Canonical bytes produced before an unsuccessful save are not a
|
||||
completion record.
|
||||
|
||||
Aggregate transcribe status still describes the whole invocation. A failed,
|
||||
running, stale, or interrupted aggregate may therefore retain reusable speaker
|
||||
records, while aggregate outputs are declared only after the current
|
||||
collection is complete. Ordinary retry reconciles and reuses those records.
|
||||
Force first checkpoints an empty current collection, preventing later retries
|
||||
from reviving pre-force results.
|
||||
|
||||
Older aggregate-only transcribe success is conservatively regenerated once
|
||||
when transcribe is selected because it does not bind individual outputs to the
|
||||
audio bytes used. The session-manifest schema identifier remains
|
||||
`narratio.session-manifest.v1`; this additive state is accepted by current
|
||||
Narratio, but older strict-decoding binaries may reject manifests after these
|
||||
fields have been written.
|
||||
|
||||
### Analyze-owned artifact state
|
||||
|
||||
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
||||
`analyze_artifacts` map keyed by normalized configured artifact key. The
|
||||
version is the authority marker: version 1 with no entries is a valid evaluated
|
||||
empty set. An absent or unsupported version is unsupported local state and is
|
||||
rejected rather than being treated as aggregate-only authority.
|
||||
|
||||
Each analyze artifact record has one disposition:
|
||||
|
||||
- `current`: the configured artifact is available and carries a versioned
|
||||
fingerprint plus a complete output record and separate output size;
|
||||
- `stale`: the recorded semantic identity is no longer current;
|
||||
- `missing`: no validated current result exists;
|
||||
- `failed`: the attempted work failed and carries a bounded diagnostic; or
|
||||
- `unselected`: the artifact was intentionally outside the evaluated set.
|
||||
|
||||
Records bind their normalized key and dependencies, fingerprint contract when
|
||||
evaluated, canonical session-relative output identity when current, producing
|
||||
Narratio run, update time, and bounded non-secret Scriptorium provenance and
|
||||
diagnostic paths. A current output includes its configured source ID, contract,
|
||||
checksum, and positive byte size. Non-current records cannot carry an output,
|
||||
so an older file is not advertised through stale, missing, failed, or
|
||||
unselected state.
|
||||
|
||||
Family-produced records additionally retain optional `family` and
|
||||
`character_id` provenance supplied by configuration resolution. These fields
|
||||
do not replace the concrete configured key or infer family membership from a
|
||||
name, so older records without them remain valid.
|
||||
|
||||
The session-stage collection is the reconciled authority across invocations.
|
||||
These records remain analyze-owned data inside the fixed stage; they are not
|
||||
dynamic stages or generic subtasks.
|
||||
|
||||
Analyze checkpoints one complete validated candidate collection after each
|
||||
successful artifact and before exposing that artifact to later work. It saves a
|
||||
detached manifest candidate and installs its version, collection, and updated
|
||||
timestamp into live state only after the store reports success. Aggregate stage
|
||||
outputs are ordinary result outputs rebuilt in configured-key order from current
|
||||
session records only.
|
||||
|
||||
Successful incremental execution replaces only evaluated artifact records and
|
||||
preserves valid unrelated current records. Rebuilt outputs are compared by
|
||||
bytes and contract: an unchanged identity permits an unselected dependent with
|
||||
the same recomputed fingerprint to remain current, while a changed identity
|
||||
removes output authority from every unselected transitive dependent by marking
|
||||
it stale. A partial analyze invocation can therefore succeed while unrelated
|
||||
configured records remain stale. Existing canonical files never create current
|
||||
records without validated execution and a successful checkpoint.
|
||||
|
||||
Aggregate analyze status is deliberately coarser than this collection. Resume
|
||||
validation may skip a succeeded aggregate record when the selected artifact
|
||||
closure is current even if unrelated records are stale. Conversely, a stale
|
||||
aggregate record may cross the ordinary runner boundary and perform zero
|
||||
Scriptorium calls when reconciliation proves every selected artifact current;
|
||||
ordinary successful stage completion then restores the aggregate status.
|
||||
|
||||
On an artifact error, analyze installs a complete retry collection on the live
|
||||
manifest before returning. The active record is failed without output, current
|
||||
transitive dependents are stale, unrelated current records survive, and only
|
||||
earlier checkpointed completions remain current. The runner's ordinary failure
|
||||
path marks aggregate analyze failed, clears aggregate outputs, invalidates
|
||||
delivery dependents, and saves that collection. A terminal session-save failure
|
||||
is joined with the original adapter, validation, filesystem, cancellation, or
|
||||
checkpoint cause.
|
||||
|
||||
If an artifact checkpoint fails, analyze rolls its working collection back to
|
||||
the last committed collection before constructing the active failed record.
|
||||
Newly materialized canonical bytes therefore remain incidental and cannot
|
||||
become current authority. A later ordinary run reconciles the durable records,
|
||||
reuses earlier current artifacts, and resumes with the failed or stale work.
|
||||
The same recovery applies after process interruption: the aggregate record may
|
||||
be non-successful, but every completed artifact checkpoint remains reusable.
|
||||
|
||||
## Remote Commit Manifest
|
||||
|
||||
`artifacts.RemoteCommitManifest` is a separate, versioned remote snapshot
|
||||
contract. It is not a serialized session manifest and contains no local
|
||||
post-publication assertion such as `current_pointer_written`. A remote commit
|
||||
identifies one campaign, session, and run and declares its immutable artifact
|
||||
set. Each artifact has a typed source, immutable destination key, SHA-256
|
||||
checksum, size, and storage generation.
|
||||
|
||||
`current/commit-pointer.json` is the sole mutable selector for the new
|
||||
contract. It identifies exactly one run-scoped `runs/{run_id}/commit.json` and
|
||||
binds that object by checksum, size, and generation. Readers strictly reject
|
||||
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||
objects that do not match their declaration.
|
||||
|
||||
An absent commit pointer means no remote state is published. Republish to
|
||||
establish a current immutable snapshot.
|
||||
|
||||
## Persistence Semantics
|
||||
|
||||
`manifest.LocalStore`:
|
||||
|
||||
- accepts only the exact supported session-manifest schema version;
|
||||
- rejects unknown fields and multiple JSON values;
|
||||
- validates stage-map identity without synthesizing missing maps, records, or
|
||||
names;
|
||||
- retains only representation-preserving collection normalization;
|
||||
- writes through a sibling temporary file and atomic replacement visibility;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
### Checkpoint persistence boundary
|
||||
|
||||
Analyze and transcribe share one private checkpoint operation in
|
||||
`internal/stage`. Their typed callers still decide when to checkpoint, validate
|
||||
their collections, populate stage-owned fields, and serialize access. The
|
||||
shared operation copies the manifest and its checkpoint-relevant mutable
|
||||
collections, applies the typed update to the copied target record, and calls
|
||||
the existing `manifest.Store` exactly once. Only after that save succeeds does
|
||||
it promote the saved target record and manifest `updated_at` into live state;
|
||||
it preserves an existing target record's identity and leaves unrelated live
|
||||
stage records in place.
|
||||
|
||||
The copy boundary includes stage records, input and artifact slices, aggregate
|
||||
output/log/configuration slices, and the typed analyze and transcribe
|
||||
collections that callbacks or `LocalStore.Save` may modify. Metadata and other
|
||||
pointer fields remain shared read-only because neither checkpoint callback nor
|
||||
the store mutates them. This is a bounded execution helper, not a general
|
||||
manifest clone or a transaction across transcript/artifact files.
|
||||
|
||||
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
||||
durability guarantee. Callers reload after a returned save error before retrying.
|
||||
A transcribe or analyze checkpoint error leaves the prior manifest collection
|
||||
as authority. The runner then attempts to persist the aggregate failure; if
|
||||
that terminal save also fails, its error is joined with the originating failure
|
||||
and the next invocation must reconcile from the last readable manifest.
|
||||
|
||||
## Execution Semantics
|
||||
|
||||
Planning and execution share one pure pre-execution decision over the recorded
|
||||
stage status, force selection, and normalized resume-validation result. That
|
||||
decision identifies run versus reuse and any required stale and dependent
|
||||
invalidation transition. Planning applies it only to a cloned manifest.
|
||||
Execution remains the sole owner of observed stage results and persistence.
|
||||
|
||||
For each selected stage, the application runner computes semantic evidence,
|
||||
performs any eligible resume validation, applies the pre-execution decision,
|
||||
and persists required invalidation before either reusing the existing success
|
||||
or beginning an attempt. The same one-stage execution owner marks the attempt
|
||||
running before invoking the stage and then records and persists its failed,
|
||||
skipped, or successful terminal state before a later stage may start. On
|
||||
success it records outputs, logs, generated configuration references, metadata,
|
||||
and—when the stage implements the optional contract—a versioned
|
||||
semantic-configuration fingerprint. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded transitive dependent session-stage records stale.
|
||||
The application owns a fixed dependency relation distinct from execution order;
|
||||
dependents are returned in canonical order. Render and extract therefore never
|
||||
stale one another, while either can stale analyze, publish, notify, and cleanup.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, metadata, and semantic fingerprint.
|
||||
Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins.
|
||||
|
||||
Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
|
||||
speaker collection and the analyze-owned per-artifact collection. This lets
|
||||
later reconciliation replace only non-current entries without erasing durable
|
||||
completions. Other stages retain aggregate-only lifecycle behavior and are
|
||||
forbidden from carrying either collection.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in the session manifest, clears older outputs,
|
||||
logs, generated configuration references, and metadata, then applies any
|
||||
bounded details from the current skip and continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs. An intentional self-skip records the current semantic
|
||||
fingerprint because it is a completed, reusable stage result; failed or
|
||||
interrupted work never promotes one.
|
||||
|
||||
When an already-succeeded stage is skipped, the session manifest deliberately
|
||||
retains its existing succeeded record because it remains the cross-invocation
|
||||
progress authority. If a stage supplies semantic configuration evidence, reuse first
|
||||
requires the persisted positive schema version and lowercase SHA-256 digest to
|
||||
match the current resolved stage semantics. Missing semantic evidence, malformed
|
||||
evidence, or a mismatch makes the stage and its fixed transitive dependents
|
||||
stale. The existing stage-specific resume
|
||||
validator runs only after this semantic check succeeds; both checks are
|
||||
required. Extraction, analyze, and cleanup have resume validators and may
|
||||
reject an otherwise eligible skip when their selected durable evidence is
|
||||
obsolete; the runner marks the aggregate record stale and executes it.
|
||||
Analyze's validator can still accept a partial selection when only unrelated
|
||||
artifact records are stale.
|
||||
|
||||
Implemented reuse coverage is deliberately split between aggregate semantic
|
||||
evidence and focused durable validators:
|
||||
|
||||
| Work | Reuse authority | Focused owners |
|
||||
| --- | --- | --- |
|
||||
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
|
||||
| transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
|
||||
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
|
||||
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
|
||||
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
|
||||
| trim | aggregate semantic fingerprint | [trim](stage-trim.md), [Scriptorium](../integrations/scriptorium.md), [Seriatim](../integrations/seriatim.md) |
|
||||
| render | aggregate semantic fingerprint | [render](stage-render.md), [Seriatim](../integrations/seriatim.md) |
|
||||
| extract | aggregate semantic fingerprint plus reference/output validator | [extract](stage-extract.md), [Notarius](../integrations/notarius.md) |
|
||||
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
|
||||
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
|
||||
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
|
||||
| cleanup | aggregate semantic fingerprint plus publish-evidence and target-absence validator | [cleanup](stage-cleanup.md), [workspace](workspace.md) |
|
||||
|
||||
These contracts record resolved choices Narratio can observe, not operational
|
||||
runner tuning. External model, module, prompt, profile, and configuration-file
|
||||
contents that a tool privately loads remain outside the contract when their
|
||||
configured identifier is unchanged; operators must force the affected work
|
||||
after such a private content change.
|
||||
|
||||
The session manifest is the authoritative stage-progress ledger across
|
||||
invocations. It retains the most recently resolved invocation's bounded
|
||||
configuration provenance. It identifies the selected profile name and source
|
||||
(`default` or `cli`) plus the effective configuration digest, but never a raw
|
||||
secret or profile content. This provenance is informational: it does not
|
||||
participate in stage resume or cache decisions. A profile change therefore
|
||||
invalidates only stages whose semantic configuration changed. When a private
|
||||
external-tool model, module, prompt, or profile changes behind an unchanged
|
||||
configured identifier, use `--force` for the affected work.
|
||||
|
||||
`session plan` computes the same current fingerprint and applies the shared
|
||||
decision and invalidation rules to a cloned manifest. It predicts the runner
|
||||
decision without persisting session state. The semantic fingerprint owner
|
||||
hashes deterministic JSON from stage-owned typed structs; stage providers must
|
||||
exclude secrets, complete effective-configuration dumps, and operational
|
||||
values that cannot affect canonical results. Concrete coverage is owned by the
|
||||
focused stage and integration documents linked above.
|
||||
|
||||
Before an explicitly bounded execution starts after `prepare`, the application
|
||||
reads the session manifest and accepts only `succeeded` or `skipped` for every
|
||||
excluded canonical prefix stage. The first other status or absent record fails
|
||||
the request before layout mutation, adapter initialization, session-manifest
|
||||
writes, or run-directory creation. Excluded prefix records are not passed to
|
||||
resume validators. Records after the selected end are not prerequisites and
|
||||
may be made stale by selected work without being scheduled.
|
||||
|
||||
After a publish commits remotely, any configured local cleanup is first recorded
|
||||
as a session-manifest obligation before deletion begins. Each target becomes
|
||||
complete only after its confined deletion (or safe absence check) and a
|
||||
successful manifest save. An incomplete obligation is retried when publish
|
||||
executes again and retains the committed run and remote identity that authorized
|
||||
it; an invocation that does not execute publish does not perform cleanup.
|
||||
|
||||
Each invocation derives campaign, session, run, local-path, and remote-prefix
|
||||
metadata from the validated resolved configuration as one projection. A persisted
|
||||
session manifest must agree on campaign and session identity before execution;
|
||||
the current projection is refreshed for every invocation while stage progress,
|
||||
inputs, and durable artifacts remain session history.
|
||||
|
||||
For handled failures after the session identity transition, the runner records
|
||||
the failure on the session ledger and atomically attempts its save while
|
||||
preserving the originating error. Abrupt process death remains an accepted case
|
||||
where the last session transition can require operator interpretation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- semantic fingerprint comparison precedes stage-specific resume validation.
|
||||
- only successful and intentional-skipped results promote current semantic
|
||||
evidence.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale succeeded stages in the fixed dependency relation.
|
||||
- the session manifest is the only durable local progress ledger.
|
||||
- remote commitment is established by a verified current pointer and remote
|
||||
commit relationship, never by a mutable session-manifest boolean.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Models and transitions: `internal/manifest/manifest.go`
|
||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
- Persistence and validation: `internal/manifest/store.go`
|
||||
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
|
||||
- Package tests: `internal/manifest/*_test.go`
|
||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||
`internal/app/run_stage_test.go`
|
||||
140
docs/internal/overview.md
Normal file
140
docs/internal/overview.md
Normal file
@@ -0,0 +1,140 @@
|
||||
# Internal Overview
|
||||
|
||||
This document is the implemented component map for Narratio. Normative system
|
||||
boundaries and dependency direction belong in
|
||||
[Architecture](../policy/architecture.md). User and operator contracts belong
|
||||
in the [CLI](../cli.md), [Configuration](../config.md),
|
||||
[Operations](../operations.md), and [Troubleshooting](../troubleshooting.md).
|
||||
Externally observable tool and format contracts belong under
|
||||
[Integrations](../integrations/).
|
||||
|
||||
## Execution Path
|
||||
|
||||
```text
|
||||
cmd/narratio -> internal/app -> configuration and production composition
|
||||
-> internal/stage -> adapters and external systems
|
||||
-> manifests and artifact resolution -> durable local/remote output
|
||||
```
|
||||
|
||||
The executable delegates process behavior to the application boundary. The
|
||||
application resolves configuration, composes concrete collaborators, acquires
|
||||
session safety controls, and runs commands. Pipeline commands execute the
|
||||
canonical stage sequence through adapter interfaces, while manifests record
|
||||
progress and artifact services resolve durable inputs and outputs.
|
||||
|
||||
The pipeline runner follows one explicit invocation lifecycle: inspect bounded
|
||||
prerequisites without mutation, prepare the workspace layout, acquire one
|
||||
session lock, load or create the authoritative session manifest, compose only
|
||||
the collaborators needed by the selected range, execute each selected stage in
|
||||
order, and then return the summary. Cleanup, when selected, is the ordinary
|
||||
final stage rather than runner finalization work.
|
||||
The lock remains held from manifest initialization through finalization, and a
|
||||
failed stage or required manifest save prevents later stages from starting.
|
||||
|
||||
## Components
|
||||
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus atomic single-file replacement, bounded reads, workspace modes, and root-scoped cleanup helpers. |
|
||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||
|
||||
The application boundary composes concrete implementations. Stages depend on
|
||||
Narratio-level contracts; external transport and SDK details remain in
|
||||
adapters. The normative rules for these relationships remain in
|
||||
[Architecture](../policy/architecture.md).
|
||||
|
||||
Command execution normalizes one private dependency value per invocation.
|
||||
Pipeline configuration-path discovery, configuration reads, secret-directory
|
||||
reads, remote downloads, object-store construction, and restore phase functions
|
||||
use that value throughout the command. Public command entry points supply the
|
||||
canonical discovery and production defaults, while focused tests may provide
|
||||
invocation-owned discovery candidates or replace only the external or sequencing
|
||||
boundary they exercise without mutating package state.
|
||||
|
||||
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
||||
model and the same pure pre-execution decision over recorded status, force, and
|
||||
normalized resume validation. Planning applies that decision and modeled
|
||||
selected-stage transitions to cloned session state; it does not persist, create
|
||||
invocation state, initialize stage-execution adapters, or claim to observe an
|
||||
execution result. Command configuration loading can still retrieve a
|
||||
missing session file through configured remote storage. It retains the initially
|
||||
composed pipeline and selected campaign while resolving either a local or
|
||||
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
||||
Analyze planning additionally exposes the artifact closure's targets,
|
||||
prerequisite rebuilds, execution order, and current reuse.
|
||||
|
||||
## Pipeline Stage Set
|
||||
|
||||
The implemented canonical order is:
|
||||
|
||||
1. [`prepare`](stage-prepare.md)
|
||||
2. [`transcribe`](stage-transcribe.md)
|
||||
3. [`merge`](stage-merge.md)
|
||||
4. [`polish`](stage-polish.md)
|
||||
5. [`normalize`](stage-normalize.md)
|
||||
6. [`trim`](stage-trim.md)
|
||||
7. [`render`](stage-render.md)
|
||||
8. [`extract`](stage-extract.md)
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (no-op)
|
||||
12. [`cleanup`](stage-cleanup.md)
|
||||
|
||||
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||
`noop` notification mode. Its versioned semantic evidence records that delivery
|
||||
mode and excludes adapter credentials and response data. The focused stage
|
||||
documents own implementation mechanics. The
|
||||
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
|
||||
and execution semantics.
|
||||
|
||||
Execution order and invalidation are separate application contracts. The stage
|
||||
registry owns the flat execution sequence. The application orchestration owner
|
||||
uses a fixed, validated dependency relation to find transitive dependents in
|
||||
canonical order. In particular, `render` and `extract` are sibling consumers of
|
||||
trimmed transcript state: neither invalidates the other, while either can stale
|
||||
`analyze`, `publish`, `notify`, and `cleanup`. Cleanup is a dependent of notify,
|
||||
so any earlier replacement can stale a prior cleanup result.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Configuration Internals](configuration.md): pipeline imports and profiles,
|
||||
field ownership, decoding, and root-relative path semantics.
|
||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
resolution, previous requirements, and current-state helpers.
|
||||
- [Manifest Internals](manifest.md): session records, persistence, and execution
|
||||
transitions.
|
||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||
guardrails.
|
||||
- [Restore Internals](command-restore.md): discovery, planning, execution, and
|
||||
reporting.
|
||||
- [`prepare`](stage-prepare.md)
|
||||
- [`transcribe`](stage-transcribe.md)
|
||||
- [`merge`](stage-merge.md)
|
||||
- [`polish`](stage-polish.md)
|
||||
- [`normalize`](stage-normalize.md)
|
||||
- [`trim`](stage-trim.md)
|
||||
- [`render`](stage-render.md)
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
- [`cleanup`](stage-cleanup.md)
|
||||
|
||||
Use this map to find an owner, then read its focused documentation and tests
|
||||
before changing behavior.
|
||||
|
||||
The stage registry is implemented in `internal/stage/placeholders.go` and its
|
||||
ordering is protected by `internal/app/planner_test.go`. Cross-invocation skip,
|
||||
force, failure, and invalidation behavior is exercised in
|
||||
`internal/app/runner_test.go` and `internal/app/run_stage_test.go`.
|
||||
214
docs/internal/stage-analyze.md
Normal file
214
docs/internal/stage-analyze.md
Normal file
@@ -0,0 +1,214 @@
|
||||
# Stage: analyze
|
||||
|
||||
## Purpose
|
||||
|
||||
Reconcile configured Scriptorium artifacts, execute only required work in
|
||||
dependency order, and safely materialize validated outputs.
|
||||
|
||||
## Inputs
|
||||
|
||||
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
||||
party artifact families have already expanded into this map during
|
||||
configuration resolution, including corresponding member dependencies and
|
||||
rewritten member-artifact input sources. Each artifact's dependency list is
|
||||
already the normalized union of explicit ordering edges and configured-artifact
|
||||
input edges.
|
||||
- optional selected artifact keys supplied through the stage environment
|
||||
- built-in, configured, extraction, and previous-session source references in
|
||||
artifact inputs
|
||||
|
||||
Supported source families:
|
||||
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
|
||||
`narratio.input.glossary`, `narratio.input.spell_catalog`
|
||||
- configured artifacts: `narratio.artifact.<key>`
|
||||
- extraction lanes: `narratio.extraction.<key>`
|
||||
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||
|
||||
## Outputs
|
||||
|
||||
- one current per-artifact manifest record per validated materialized output
|
||||
- stage metadata describing selected/generated/reused artifacts
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- when `pipeline.scriptorium` is absent or no configured artifact is
|
||||
executable, completes successfully and records explanatory metadata. Before
|
||||
returning, it reconciles and checkpoints the complete session collection:
|
||||
records removed from configuration are dropped, while valid records for
|
||||
configured but unselected artifacts remain authoritative. This is not an
|
||||
explicit self-skip: the session manifest records success, satisfies publish's
|
||||
prerequisite, and an ordinary later run reuses the result while the effective
|
||||
set remains empty. Enabling or selecting an artifact later makes missing
|
||||
versioned evidence non-resumable and schedules it without requiring force.
|
||||
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||
and configured extraction lanes. Extraction availability is hydrated only
|
||||
from compatible successful extraction evidence.
|
||||
- uses enabled configured artifacts by default. An explicit `--artifacts`
|
||||
selection is a one-invocation override that makes exactly the named
|
||||
configured artifacts explicit targets even when disabled. The work planner
|
||||
adds required configured prerequisites, reuses current ones, and schedules
|
||||
stale, missing, or otherwise non-current prerequisites before dependents.
|
||||
- makes a non-executable configured artifact reusable only when its current
|
||||
manifest record and durable output pass the configured-artifact evidence
|
||||
contract; an incidental or stale canonical file is unavailable.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- consumes only the concrete dependency graph finalized by configuration;
|
||||
planning, prerequisite closure, execution, reuse, invalidation, and semantic
|
||||
evidence do not infer a second graph from inputs.
|
||||
- resolves required/optional inputs per artifact source definition into an
|
||||
ordered semantic identity. Each identity records the configured input name,
|
||||
canonical source ID, required policy, explicit presence, source contract,
|
||||
checksum, size, and a source-based logical identity. Workspace paths and
|
||||
producer run IDs are excluded.
|
||||
- orders input identities by configured input name independently of Go map
|
||||
iteration. Runtime adapter paths remain a separate execution-only map.
|
||||
- omits an unavailable optional input from the adapter request while retaining
|
||||
explicit absence in its semantic identity; an unavailable required input
|
||||
fails.
|
||||
- resolves prepared stable input sources through the shared manifest-authoritative
|
||||
identity resolver; it does not accept incidental files or fall back to
|
||||
campaign/session source paths.
|
||||
- reuses checksums and sizes from validated prepared, extraction, and current
|
||||
configured-artifact evidence. Other resolved inputs are hashed as confined
|
||||
regular files with streaming reads and the central resolved-artifact size
|
||||
limit.
|
||||
- owns a versioned SHA-256 fingerprint contract with one fixed-field canonical
|
||||
JSON payload and no map serialization. Configured artifacts are fingerprinted
|
||||
in deterministic dependency order.
|
||||
- fingerprints the normalized artifact key, prompt and profile identifiers,
|
||||
effective render-debug behavior, session-relative output identity, sorted
|
||||
dependency keys, ordered input declarations and semantic identities,
|
||||
validated current dependency-output identities, and sorted effective
|
||||
Scriptorium variables (including Narratio's sticky session variable).
|
||||
- provides read-only reconciliation that classifies each configured record as
|
||||
current, stale, missing, failed, or otherwise non-resumable, and
|
||||
separately identifies manifest records removed from current configuration.
|
||||
A record is current only when its fingerprint version and value match and its
|
||||
configured output still passes manifest-authoritative evidence validation.
|
||||
- owns a read-only typed work planner. Its explicit targets are enabled
|
||||
artifacts by default or the exact normalized `--artifacts` selection when
|
||||
supplied. It closes targets over configured prerequisites, orders the closure
|
||||
topologically, reuses current members, and schedules every non-current member
|
||||
before its dependents.
|
||||
- force applies only to explicit targets. A current prerequisite is reused
|
||||
unless it is itself an explicit forced target; disabled prerequisites may be
|
||||
rebuilt when required, while unrelated disabled artifacts are excluded.
|
||||
- the work plan carries explicit targets, prerequisite-only work, deterministic
|
||||
execution and reuse lists, invalidated and removed records, and a cloned
|
||||
reconciled starting collection. Valid unrelated configured records survive
|
||||
reconciliation, removed records are omitted, and incidental files never
|
||||
become current without manifest-backed regeneration.
|
||||
- implements aggregate resume validation by running the same read-only catalog,
|
||||
fingerprint reconciliation, and work planner used by execution. A succeeded
|
||||
aggregate record is reusable exactly when the selected closure schedules no
|
||||
artifact work and no removed configuration records require a checkpoint;
|
||||
stale unrelated records do not block a partial selection.
|
||||
- exposes the typed artifact decision to `session plan`. Planning applies it to
|
||||
a cloned manifest after modeling earlier selected stage transitions, so
|
||||
aggregate run/skip and artifact execute/reuse decisions match the ordinary
|
||||
runner without creating durable state or invoking Scriptorium.
|
||||
- executes only the work plan's scheduled entries. Manifest-validated current
|
||||
prerequisites remain available through the runtime catalog without invoking
|
||||
Scriptorium; newly produced prerequisites enter that catalog with the same
|
||||
contract, checksum, and size identity used for persisted current evidence.
|
||||
- before the first adapter call, checkpoints the reconciled starting collection
|
||||
when it removed configuration records or cleared obsolete authority. After
|
||||
each successful artifact, checkpoints the complete collection before making
|
||||
that artifact available to the next scheduled dependent. Analyze supplies
|
||||
its typed update to the shared
|
||||
[checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary)
|
||||
and refreshes its working and committed collections only after the save
|
||||
succeeds.
|
||||
- keeps adapter output in the invocation's run-local analyze directory until
|
||||
it is a safe, non-empty, bounded regular file with a calculated checksum and
|
||||
complete output contract. Canonical replacement uses the shared atomic file
|
||||
operation boundary and verifies that the installed checksum matches the
|
||||
validated run-local bytes.
|
||||
- records each successful artifact's freshly computed fingerprint, canonical
|
||||
relative output path, contract, checksum, size, producer run ID, bounded
|
||||
Scriptorium provenance, logs, and generated configuration references in the
|
||||
analyze-owned manifest collection.
|
||||
- preserves valid unrelated current records during partial execution. If a
|
||||
rebuilt output's bytes and contract are unchanged, unselected dependents may
|
||||
remain current. If that semantic identity changes, unselected transitive
|
||||
dependents become stale without being executed; dependents included in the
|
||||
invocation are evaluated in dependency order instead.
|
||||
- returns aggregate outputs from every current session record in normalized-key
|
||||
order after the complete reconciled collection has been checkpointed;
|
||||
unrelated stale records do not make an otherwise successful partial
|
||||
invocation fail.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
- validates non-empty output files and materializes canonical outputs.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- required missing configured/previous-session inputs fail.
|
||||
- missing required prepared stable input source includes prepare rerun guidance.
|
||||
- missing required previous-session source includes prepare rerun guidance.
|
||||
- missing required `narratio.transcript.final_markdown` or
|
||||
`narratio.transcript.final_trimmed_markdown` inputs includes render rerun
|
||||
guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
- a scheduled artifact failure installs a retry collection on the live manifest
|
||||
with the active artifact marked `failed`, a bounded error, and no output
|
||||
authority. Current transitive dependents become stale without execution.
|
||||
- earlier artifacts remain current only after their run-local output passed
|
||||
validation, canonical materialization, and a successful complete-collection
|
||||
manifest checkpoint; unattempted later artifacts retain their reconciled
|
||||
session state.
|
||||
- unrelated current records survive a partial failure. Old canonical bytes for
|
||||
the failed artifact and newly materialized bytes whose checkpoint fails are
|
||||
incidental, not current evidence.
|
||||
- a checkpoint failure stops the queue immediately. The in-memory retry
|
||||
collection starts from the last successful checkpoint, marks the active
|
||||
artifact failed without output authority, and stales its current transitive
|
||||
dependents.
|
||||
- the runner's ordinary error path marks aggregate analyze failed, clears its
|
||||
aggregate outputs, invalidates publish, notify, and cleanup through the
|
||||
application dependency relation, and persists the live retry collection.
|
||||
Terminal-save errors are joined with the original artifact or checkpoint
|
||||
failure.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- input-identity resolution is read-only: it does not invoke adapters,
|
||||
materialize outputs, update status, or create run-local files.
|
||||
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
||||
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
||||
executable or private transitive configuration contents. A change that is
|
||||
visible only inside Scriptorium—such as a file privately loaded by its config
|
||||
path—requires an explicit forced regeneration.
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
- a canonical file without current per-artifact manifest evidence is never
|
||||
promoted to current state.
|
||||
- an ordinary retry after interruption or partial failure reuses every valid
|
||||
earlier checkpoint and schedules only failed, stale, missing, or otherwise
|
||||
non-current work in dependency order.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#scriptorium-artifact-entries) owns artifact
|
||||
fields and source-selection rules, including
|
||||
[artifact families](../config.md#scriptorium-artifact-families).
|
||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||
- Implementation and tests: `internal/stage/analyze.go`,
|
||||
`internal/stage/analyze_artifact_execution.go`,
|
||||
`internal/stage/analyze_execution_state.go`,
|
||||
`internal/stage/analyze_input_identity.go`,
|
||||
`internal/stage/analyze_fingerprint.go`,
|
||||
`internal/stage/analyze_reconciliation.go`, `internal/stage/analyze_plan.go`,
|
||||
`internal/stage/analyze_resume.go`, `internal/stage/analyze_test.go`,
|
||||
`internal/stage/analyze_checkpoint_test.go`,
|
||||
`internal/stage/analyze_input_identity_test.go`,
|
||||
`internal/stage/analyze_fingerprint_test.go`,
|
||||
`internal/stage/analyze_reconciliation_test.go`,
|
||||
`internal/stage/analyze_plan_test.go`,
|
||||
`internal/stage/analyze_incremental_execution_test.go`,
|
||||
`internal/stage/analyze_failure_test.go`,
|
||||
`internal/stage/analyze_resume_test.go`, and
|
||||
`internal/app/analyze_checkpoint_recovery_test.go`.
|
||||
54
docs/internal/stage-cleanup.md
Normal file
54
docs/internal/stage-cleanup.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# Stage: cleanup
|
||||
|
||||
## Purpose
|
||||
|
||||
Remove explicitly configured local data for the run selected by a successful
|
||||
publish commit. Cleanup is the twelfth and final canonical stage, after
|
||||
`notify`.
|
||||
|
||||
## Authorization And Targets
|
||||
|
||||
Cleanup is disabled unless at least one of these policies is true:
|
||||
|
||||
- `pipeline.spool.delete_audio_after_publish`
|
||||
- `pipeline.workspace.cleanup_after_publish`
|
||||
|
||||
An enabled cleanup requires the manifest's publish stage to be succeeded with
|
||||
`uploaded=true`, a canonical `published_run_id`, and the exact remote commit and
|
||||
current commit-pointer keys for the manifest campaign/session prefix. The
|
||||
configured campaign and session must match the manifest. Cleanup derives its
|
||||
only possible targets through the canonical artifact path helpers:
|
||||
|
||||
- the published run's session spool-audio directory; and
|
||||
- the published run's workspace run directory.
|
||||
|
||||
It does not scan parent directories, infer the newest run, or read object
|
||||
storage. Every deletion uses the root-confined file operation, which rejects a
|
||||
root itself, traversal, symlink escape, and out-of-root targets.
|
||||
|
||||
## Execution And Resume
|
||||
|
||||
Disabled cleanup records an explicit skipped result. Enabled cleanup removes
|
||||
targets in deterministic policy order and treats an already-missing target as
|
||||
success. A successful record contains the published run ID and completed policy
|
||||
names.
|
||||
|
||||
A previous cleanup success is reusable only while its semantic configuration
|
||||
matches and every selected target remains absent. A recreated target makes the
|
||||
stage non-resumable. A failure is recorded against cleanup; publish remains
|
||||
succeeded and remote current state is unchanged. On a later normal full run,
|
||||
valid upstream stages are skipped and cleanup is retried without republishing.
|
||||
|
||||
`narratio publish` selects publish only and never invokes cleanup. A bounded run
|
||||
through `notify` or any earlier endpoint also excludes it. Operators may run
|
||||
`narratio run-stage cleanup <session_id>` directly; missing, failed, malformed,
|
||||
or identity-mismatched publication evidence fails before deletion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Planning and evidence validation: `internal/stage/cleanup_plan.go`
|
||||
- Execution and resume: `internal/stage/cleanup.go`
|
||||
- Root-confined deletion: `internal/fileops/cleanup.go`
|
||||
- Focused tests: `internal/stage/cleanup_plan_test.go`,
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
148
docs/internal/stage-extract.md
Normal file
148
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# Internal: Extract Stage
|
||||
|
||||
## Responsibility
|
||||
|
||||
`extract` runs after `render` and before `analyze`. It converts the canonical
|
||||
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||
|
||||
The external protocol is documented in the
|
||||
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||
fields belong in [Configuration](../config.md), and physical paths and force
|
||||
procedures belong in [Operations](../operations.md).
|
||||
|
||||
## Lifecycle
|
||||
|
||||
`internal/stage/extract.go`:
|
||||
|
||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||
2. resolves every configured prepared reference through the shared
|
||||
manifest-authoritative identity resolver before creating run-local output;
|
||||
3. resolves each reference to a stable content-addressed session snapshot,
|
||||
reusing verified bytes or reconstructing a missing/corrupt regular entry,
|
||||
and rejects any source change observed while copying;
|
||||
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
||||
including sorted reference identities and the explicit downstream
|
||||
configuration-profile selection;
|
||||
5. creates a run-local staging directory and invokes the injected
|
||||
`notarius.Runner`, carrying the optional already-resolved credential
|
||||
reference without exposing its value, the optional exact configuration
|
||||
profile, plus effective resume and chunk-cache controls after force
|
||||
precedence;
|
||||
6. revalidates the reference snapshots, then validates the v2 successful
|
||||
receipt, confined index, management documents, configured required lane
|
||||
descriptors, validation summaries, and regular payload files;
|
||||
7. atomically promotes the complete bundle to its immutable durable location;
|
||||
8. records one non-selectable `notarius_index` output and one selectable
|
||||
`notarius_lane` output per configured lane; and
|
||||
9. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||
Scriptorium and publish resolution.
|
||||
|
||||
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||
Narratio run ID, the resolved trimmed-input identity, and invocation
|
||||
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||
source ID, producer stage/output/run identity, and resolution provenance.
|
||||
Reference metadata contains only selector, source ID, canonical session-relative
|
||||
path, checksum, and size; adapter requests receive selector and absolute stable
|
||||
snapshot path, never payload contents. Snapshot bytes must
|
||||
match the prepared identity both before and after Notarius runs, so a concurrent
|
||||
prepared-file replacement cannot make recorded provenance describe different
|
||||
bytes from those supplied to Notarius.
|
||||
Validation completes before the single durable rename, so a rejected result
|
||||
cannot expose a partial durable bundle.
|
||||
|
||||
Snapshot destinations are owned by `internal/artifacts` under
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
|
||||
or timestamp and can be reused across invocations. Identical bytes and basename
|
||||
share a verified file; changed bytes select a different digest directory.
|
||||
Unsafe path components and non-regular conflicts fail without recursive
|
||||
deletion. Snapshot paths are delivery details and never become manifest output
|
||||
evidence or prerequisites for reuse of an already complete extract bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded analysis and delivery dependents stale. Render is an independent
|
||||
sibling and remains current. Repeating the same disabled self-skip with no
|
||||
outputs is stable and does not repeatedly invalidate dependent stages.
|
||||
|
||||
## Resume Validation
|
||||
|
||||
Before the focused validator runs, the application compares extract's versioned
|
||||
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
||||
reference selector/source mappings, sorted declared output contracts, and each
|
||||
canonical `narratio.extraction.<key>` output identity. It also covers the
|
||||
explicit Notarius configuration-profile selection; omitted, empty, and exact
|
||||
`default` selectors share the base-selection identity. It excludes executable,
|
||||
timeout, working directory, config path, and private Notarius config contents.
|
||||
Credential presence, reference name, and resolved value are likewise
|
||||
operational-only and excluded. Ordinary command loading still resolves a
|
||||
selected reference before resume evaluation, but a credential-only change does
|
||||
not rerun a valid completed extraction or stale its dependents.
|
||||
|
||||
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
||||
stage record still matches the current byte- and provenance-bearing invocation
|
||||
evidence. That evidence covers the current direct trimmed-transcript identity,
|
||||
sorted prepared-reference identities, pipeline identity, and configured output
|
||||
contracts, plus the same downstream profile-selection identity. The same
|
||||
reference helper and transcript identity are resolved again for artifact
|
||||
evidence, so changing current transcript bytes, reference bytes, producer
|
||||
identity, or a named downstream selection makes the prior extraction obsolete.
|
||||
Operational runner settings do not invalidate otherwise current durable
|
||||
evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
|
||||
make a complete extraction non-resumable. If extraction executes, ordinary
|
||||
runs forward the configured values, while force disables resume and selects
|
||||
`refresh`.
|
||||
|
||||
A valid prepared-reference change makes extraction non-resumable. Missing,
|
||||
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
||||
with prepare-force guidance because an immediate extract rerun cannot succeed.
|
||||
|
||||
The validator then checks the producing run identity, canonical immutable
|
||||
bundle root, path confinement and absence of symlink components, receipt
|
||||
identity, exactly one canonical index, the exact configured source set,
|
||||
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||
return an error rather than silently accepting or replacing data.
|
||||
|
||||
This sensitivity is limited to the explicit downstream selector. Narratio's
|
||||
own profile label and whole effective-configuration digest are not blanket
|
||||
extraction invalidators. Neither contract can observe files imported by
|
||||
Notarius configuration, profile contents, prompt/module definitions, or other
|
||||
transitive inputs. Operators must force extraction after changing any such
|
||||
private input behind a stable identifier.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, inconsistent warning or diagnostic envelopes,
|
||||
required-lane rejection or incomplete validation, payload inspection,
|
||||
checksum, or installation errors fail the stage through ordinary manifest
|
||||
transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and installation.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier immutable promoted bundle remains available for audit and
|
||||
recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||
`internal/stage/extract_resume.go`,
|
||||
`internal/stage/extract_test.go`,
|
||||
`internal/stage/semantic_contracts_delivery.go`
|
||||
- Stable snapshot materialization and verification:
|
||||
`internal/stage/extract_references.go`
|
||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||
`internal/adapters/notarius/subprocess_test.go`
|
||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`
|
||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||
- Opt-in Notarius v0.7.0 production-boundary coverage:
|
||||
`internal/stage/notarius_cache_integration_test.go`
|
||||
57
docs/internal/stage-merge.md
Normal file
57
docs/internal/stage-merge.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Stage: merge
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/raw/*.json`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- optional `artifacts/seriatim.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers and validates raw transcript inputs. For versioned transcribe
|
||||
state, discovery requires aggregate transcribe success and consumes only the
|
||||
aggregate outputs that exactly match the current per-speaker collection.
|
||||
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
|
||||
workflows without versioned speaker state retain directory discovery.
|
||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||
- validates merged transcript and optional report JSON.
|
||||
- materializes canonical outputs and records stage logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- merge always consumes normalized forms of raw inputs.
|
||||
- a versioned transcribe collection is usable only through its complete,
|
||||
successful aggregate output declaration.
|
||||
- base transcript must validate before stage success.
|
||||
- report output is config-gated.
|
||||
|
||||
## Resume Evidence
|
||||
|
||||
Merge records a versioned semantic-configuration fingerprint for the Seriatim
|
||||
merge operation, output schema, coalesce gap, and every configured advanced
|
||||
merge transformation. A change reruns merge and stales only its fixed
|
||||
descendants; prepare and transcribe remain reusable. Binary path, timeout,
|
||||
report emission, logs, and diagnostic retention are operational exclusions.
|
||||
|
||||
Configuration or resources loaded privately inside Seriatim are outside
|
||||
Narratio's observable contract and require `--force` when changed. An existing
|
||||
successful merge record without evidence reruns once when selected.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Seriatim values.
|
||||
- Implementation and tests: `internal/stage/merge.go`,
|
||||
`internal/stage/merge_test.go`,
|
||||
`internal/stage/semantic_contracts_initial.go`, and
|
||||
`internal/stage/semantic_contracts_initial_test.go`
|
||||
42
docs/internal/stage-normalize.md
Normal file
42
docs/internal/stage-normalize.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Stage: normalize
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.json` (or configured normalize output path)
|
||||
- optional `artifacts/seriatim.normalize.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves polished transcript only from verified manifest output evidence.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
- validates normalized transcript and optional report.
|
||||
- materializes canonical outputs and records logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- final transcript must validate as processed transcript JSON (`segments` array).
|
||||
- normalize defaults are applied when `pipeline.normalize` is unset.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers the Seriatim normalize operation,
|
||||
output schema and canonical output identity, plus the configured transcript
|
||||
transformations. Seriatim's executable and timeout and optional report
|
||||
generation are operational and do not invalidate the normalized transcript.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns normalize fields and defaults.
|
||||
- Implementation and tests: `internal/stage/normalize.go`,
|
||||
`internal/stage/normalize_test.go`,
|
||||
`internal/stage/semantic_contracts_refinement.go`
|
||||
54
docs/internal/stage-polish.md
Normal file
54
docs/internal/stage-polish.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# Stage: polish
|
||||
|
||||
## Purpose
|
||||
|
||||
Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- `inputs/glossary.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
- optional `artifacts/audita.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves base transcript only from verified merge manifest output evidence.
|
||||
- invokes an Audita runner configured with static model/runtime options; the
|
||||
invocation supplies paths and modules.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
- validates optional report JSON.
|
||||
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||
|
||||
## Invariants
|
||||
|
||||
- polished transcript schema validation is mandatory.
|
||||
- report output is config-gated.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers the Audita service endpoint, model,
|
||||
validation model, module set, transcript description, output schema, selected
|
||||
external configuration path, and canonical polished-transcript identity. Module
|
||||
ordering is normalized because the configured modules form a set. Audita's
|
||||
executable, timeouts, concurrency, report and debug behavior, work retention,
|
||||
and resolved credential value are operational and do not invalidate a
|
||||
successful result. The public credential reference name may participate in
|
||||
configuration provenance but never exposes its resolved value.
|
||||
|
||||
Narratio can fingerprint a selected model, module, or configuration identifier,
|
||||
but it cannot inspect content that Audita privately resolves behind that stable
|
||||
identifier. Force `polish` after changing such private content without changing
|
||||
its identifier.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Audita](../integrations/audita.md) owns subprocess, validation, and failure
|
||||
semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Audita values.
|
||||
- Implementation and tests: `internal/stage/polish.go`,
|
||||
`internal/stage/polish_test.go`,
|
||||
`internal/stage/semantic_contracts_refinement.go`
|
||||
99
docs/internal/stage-prepare.md
Normal file
99
docs/internal/stage-prepare.md
Normal file
@@ -0,0 +1,99 @@
|
||||
# Stage: prepare
|
||||
|
||||
## Purpose
|
||||
|
||||
Materialize canonical current-session inputs before processing stages.
|
||||
|
||||
## Inputs
|
||||
|
||||
- resolved campaign, session, and pipeline configuration
|
||||
- stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
|
||||
- optional spell-catalog overlay
|
||||
- one resolved local or S3 audio source
|
||||
- enabled configured artifact input requirements for previous-session sources
|
||||
|
||||
## Outputs
|
||||
|
||||
- `inputs/campaign.yml`
|
||||
- `inputs/session.yml`
|
||||
- `inputs/pipeline.resolved.yml`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
- `inputs/glossary.yml`
|
||||
- `inputs/players.yml`
|
||||
- `inputs/party.yml`
|
||||
- optional `inputs/spell_catalog.json`
|
||||
- `audio/*.flac`
|
||||
- optional `previous/manifest.json`
|
||||
- optional `previous/artifacts/**`
|
||||
- deterministic `manifest.inputs` entries (checksums + provenance)
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- validates required config/store state.
|
||||
- enforces local audio vs S3 audio mutual exclusivity.
|
||||
- rejects duplicate explicit local audio sources after resolution.
|
||||
- gives distinct local source paths with the same basename deterministic unique
|
||||
prepared filenames so neither source is overwritten.
|
||||
- materializes S3 audio through spool/cache-aware logic.
|
||||
- materializes a configured spell catalog with checksum and provenance, or
|
||||
safely removes an obsolete canonical spell catalog and its manifest record
|
||||
when the effective input is omitted.
|
||||
- copies the validated canonical party bytes unchanged and deterministically
|
||||
generates the prepared players projection.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||
- resolves the pointer-selected previous source through the shared resolver;
|
||||
- downloads previous manifest/artifacts;
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||
are typed skipped results. Each requirement resolves only from its unique
|
||||
`published_output` declaration in the pointer-selected remote commit. Prepare
|
||||
strips that selected run's immutable `published/` prefix to derive the local
|
||||
cache path, then validates the committed checksum, size, and generation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `prepare` hydrates canonical `previous/` cache state.
|
||||
- managed previous artifacts are stored under `previous/artifacts/**` without
|
||||
duplicate `artifacts/artifacts/` nesting.
|
||||
- managed `previous/` state represents only the current requirement set.
|
||||
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||
|
||||
## Resume Evidence
|
||||
|
||||
Prepare records a versioned semantic-configuration fingerprint for the
|
||||
resolved campaign/session selection, local-versus-S3 audio mode and canonical
|
||||
audio names, stable-input ownership/presence, previous-session identity, and
|
||||
the party mode plus canonical players projection version, and the effective
|
||||
previous-artifact requirement set. A change reruns prepare and
|
||||
stales its fixed descendants. Existing successful records without this
|
||||
evidence rerun once when selected.
|
||||
|
||||
Workspace, spool, and cache placement and absolute source relocation are not
|
||||
semantic when logical selection, canonical names, and bytes are equivalent.
|
||||
The fingerprint deliberately does not read or rehash large audio. Prepared
|
||||
input checksums remain the content provenance. Before reusing success, prepare
|
||||
validates every durable prepared copy and compares current stable-input bytes,
|
||||
canonical party and derived-player bytes, local audio membership/checksums, or
|
||||
S3 key/size/entity-tag identity with that provenance. Source relocation with
|
||||
equivalent names and bytes remains reusable; changed or unavailable evidence
|
||||
causes a normal prepare rerun.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md) owns audio selection, stable input fields, and
|
||||
previous-session settings.
|
||||
- [Operations](../operations.md) owns physical input, audio, spool, cache, and
|
||||
previous-state layout.
|
||||
- [Storage Internals](storage.md) and [Artifact Internals](artifacts.md) explain
|
||||
the internal collaborators.
|
||||
- Implementation and tests: `internal/stage/prepare.go`,
|
||||
`internal/stage/prepare_test.go`,
|
||||
`internal/stage/prepare_resume.go`,
|
||||
`internal/stage/prepare_resume_test.go`,
|
||||
`internal/stage/semantic_contracts_initial.go`,
|
||||
`internal/stage/semantic_contracts_initial_test.go`,
|
||||
`internal/audio/s3_audio_test.go`,
|
||||
`internal/previouscache/*_test.go`
|
||||
147
docs/internal/stage-publish.md
Normal file
147
docs/internal/stage-publish.md
Normal file
@@ -0,0 +1,147 @@
|
||||
# Stage: publish
|
||||
|
||||
## Purpose
|
||||
|
||||
Upload run/session outputs to object storage and atomically advance remote current state.
|
||||
|
||||
## Inputs
|
||||
|
||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||
- declared run-local diagnostic files
|
||||
- resolved publish output rules
|
||||
- independent per-source remote lock objects read at planning start
|
||||
- the validated prior current commit when an initially locked source may carry
|
||||
- durable previous-session cache files when present
|
||||
|
||||
## Outputs
|
||||
|
||||
- uploaded session manifest, declared run-local diagnostics, and selected
|
||||
publish outputs;
|
||||
- uploaded durable previous-session cache files when present;
|
||||
- immutable run-scoped commit manifest; and
|
||||
- current commit pointer, written last.
|
||||
|
||||
Exact remote placement and the operator workflow belong in
|
||||
[Operations](../operations.md#publish-workflow).
|
||||
|
||||
## Key Behavior
|
||||
|
||||
`publish.go` retains the transaction-owning `Run` path. Source discovery,
|
||||
manifest-derived archive metadata, confined opening, and checksum validation
|
||||
live in `publish_sources.go`; artifact-catalog matching, selection, required
|
||||
and optional rules, destination resolution, and collision inputs live in
|
||||
`publish_outputs.go`. Copying, hashing, immutable upload, reuse checks, and
|
||||
remote verification live in `publish_transfer.go`; committed-manifest,
|
||||
immutable-commit, and current-pointer encoding and writes live in
|
||||
`publish_commit.go`. `publish_output_plan.go` owns the deterministic, I/O-free
|
||||
five-outcome model and its secret-free metadata projection.
|
||||
|
||||
- when publishing or run upload is disabled, completes successfully with no
|
||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||
the session manifest records success. Enablement and upload policy are
|
||||
fingerprinted, so changing either automatically makes the prior result
|
||||
non-resumable.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- derives a deterministic run-archive allowlist from the session manifest's
|
||||
declared current-run outputs, logs, and generated configurations. References
|
||||
outside the current run root, including canonical session files and prior-run
|
||||
diagnostics, are not archive candidates. No distinguished diagnostic file is
|
||||
read or uploaded.
|
||||
- opens each archive candidate beneath its archive root without following
|
||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||
checks a declared checksum when present, then streams the opened descriptor.
|
||||
- derives the durable previous-cache archive from its validated manifest using
|
||||
the same confinement and regular-file checks.
|
||||
- resolves publish output sources through runtime artifact catalog and
|
||||
manifest-aware resolution. Configured Scriptorium outputs are publishable
|
||||
only from validated `current` per-artifact analyze evidence; an incidental
|
||||
canonical file, aggregate output, stale/failed/unselected record, or
|
||||
mismatched path, size, or checksum remains unavailable. Built-in and
|
||||
extraction sources likewise require their own current manifest evidence.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- plans every configured output into exactly one source-sorted outcome:
|
||||
`published`, `carried_locked`, `locked_unpublished`, `optional_missing`, or
|
||||
`unselected`.
|
||||
- an initially locked selected output carries its exact prior
|
||||
`published_output` declaration when one exists. The declaration may point to
|
||||
an earlier run in the same session and retains its source, key, checksum,
|
||||
size, and generation; the bytes are not copied.
|
||||
- verifies a carried object's exact key, size, and generation through object
|
||||
metadata without downloading or opening the local output source.
|
||||
- an initially locked source with no prior declaration remains absent even
|
||||
when the output rule is required. Optional missing unlocked outputs remain
|
||||
absent; required missing unlocked outputs fail.
|
||||
- creates one complete immutable source-to-destination mapping before upload;
|
||||
- uploads and verifies every declared immutable object and the commit manifest;
|
||||
- writes the immutable commit from the combined new and exact carried
|
||||
declarations.
|
||||
- rereads every initially unlocked participating source after the immutable
|
||||
commit. A newly present lock aborts selection, leaving the uploaded immutable
|
||||
attempt unselected. Removing an initial lock may conservatively produce one
|
||||
carry because the initial snapshot controls the plan.
|
||||
- updates `current/commit-pointer.json` exactly once, last, and writes no other
|
||||
mutable publication control record.
|
||||
|
||||
## Metadata Signals
|
||||
|
||||
Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
- previous uploads
|
||||
- one `publish_output_outcomes` collection containing all five output states,
|
||||
exact declared keys, and key-derived origin runs where applicable
|
||||
- remote commit and current-pointer key paths
|
||||
- the run identifier selected by the commit
|
||||
|
||||
## Invariants
|
||||
|
||||
- `current/commit-pointer.json` is the remote commit marker and is written last.
|
||||
- run files, selected outputs, previous-cache files, and the committed session
|
||||
manifest are all declared by an immutable commit under the run prefix.
|
||||
- run and previous uploads contain only manifest-declared regular files opened
|
||||
from verified descriptors; symlinks, special files, replacement races, and
|
||||
undeclared entries are rejected or ignored before uploads begin.
|
||||
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||
archived only when recorded by the session manifest. A run with no declared
|
||||
current-run diagnostics legitimately uploads none.
|
||||
- independent publish locks are not overridden by `--force`; every initially
|
||||
unlocked relevant source is reread immediately before current-state
|
||||
selection.
|
||||
- successful uploaded publication records the exact run ID, remote commit key,
|
||||
and current-pointer key consumed by the later cleanup stage.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
|
||||
normalized source/destination/required output rules, and the remote backend,
|
||||
bucket, region, endpoint, and root-prefix identity. Rule ordering is
|
||||
canonicalized. Credential
|
||||
environment names, path-addressing transport mode, local workspace placement,
|
||||
and run identifiers are excluded. Independent remote locks remain mutable state
|
||||
and are read directly by publish at planning and commit boundaries; semantic
|
||||
evidence does not replace those safety checks.
|
||||
|
||||
The commit boundary and cleanup authorization are normative architecture invariants; see
|
||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||
|
||||
Publish never deletes local state. The standalone `publish` command selects
|
||||
only this stage; a full run selects notify and cleanup afterward.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output
|
||||
rules.
|
||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||
physical remote state.
|
||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||
helpers.
|
||||
- Implementation and tests: `internal/stage/publish.go`,
|
||||
`internal/stage/publish_sources.go`, `internal/stage/publish_outputs.go`,
|
||||
`internal/stage/publish_output_plan.go`,
|
||||
`internal/stage/publish_transfer.go`, `internal/stage/publish_commit.go`,
|
||||
`internal/stage/publish_test.go`,
|
||||
`internal/stage/semantic_contracts_delivery.go`,
|
||||
`internal/app/operator_helpers_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
60
docs/internal/stage-render.md
Normal file
60
docs/internal/stage-render.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Stage: render
|
||||
|
||||
## Purpose
|
||||
|
||||
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
|
||||
It runs after `trim` and before `extract` in the canonical sequence. Render and
|
||||
extract are independent sibling consumers: replacing render output does not
|
||||
invalidate extraction, but it does invalidate succeeded analysis and delivery
|
||||
records that may consume rendered transcripts.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `narratio.transcript.final` (`transcripts/final.json`)
|
||||
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
|
||||
|
||||
## Outputs
|
||||
|
||||
- `narratio.transcript.final_markdown` -> `transcripts/final.md`
|
||||
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||
- resolves inputs only from verified manifest output evidence.
|
||||
- writes run-local outputs first, then materializes canonical session outputs.
|
||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||
and records explanatory metadata. This is not an explicit self-skip: both
|
||||
manifests record success. Because enablement is fingerprinted, enabling
|
||||
render later automatically makes the prior result non-resumable.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- missing normalized input fails with normalize rerun guidance.
|
||||
- missing trimmed input fails with trim rerun guidance.
|
||||
- adapter/subprocess failure fails stage.
|
||||
- empty render output files fail validation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `format: markdown` is supported.
|
||||
- render stage owns production of built-in Markdown transcript sources.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers enablement, final format, resolved
|
||||
title (including the session-title fallback), timestamp, segment-ID and
|
||||
metadata inclusion, both canonical input identities, and both Markdown output
|
||||
identities. Seriatim's executable, timeout, and report behavior are operational
|
||||
and do not invalidate rendered transcripts. A render-only change leaves the
|
||||
independent `extract` sibling reusable while invalidating their shared
|
||||
downstream consumers.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns render subprocess behavior.
|
||||
- [Configuration](../config.md#pipeline) owns render fields and defaults.
|
||||
- Implementation and tests: `internal/stage/render.go`,
|
||||
`internal/stage/render_test.go`,
|
||||
`internal/stage/semantic_contracts_refinement.go`
|
||||
94
docs/internal/stage-transcribe.md
Normal file
94
docs/internal/stage-transcribe.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Stage: transcribe
|
||||
|
||||
## Purpose
|
||||
|
||||
Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `audio/*.flac` from `prepare`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/raw/<speaker>.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers prepared audio from manifest inputs or the canonical audio
|
||||
directory and derives each speaker identity from its `.flac` filename.
|
||||
- hashes the current audio and reconciles it with the versioned per-speaker
|
||||
completion collection before starting HTTP work.
|
||||
- reuses only records whose audio identity, observable WhisperX semantics,
|
||||
canonical output location, JSON, checksum, and size remain current.
|
||||
- durably checkpoints the reconciled collection before dispatch, then sends
|
||||
only missing or obsolete speakers through the bounded worker group.
|
||||
- validates each successful run-local response, rechecks the audio bytes,
|
||||
materializes the canonical raw transcript, and checkpoints that speaker
|
||||
before accepting another completion.
|
||||
- rebuilds aggregate stage outputs in stable speaker order from the complete
|
||||
current collection, including records produced by earlier invocations.
|
||||
|
||||
## Invariants
|
||||
|
||||
- prepared audio identities must be unique; prepare disambiguates distinct
|
||||
source paths that share a basename.
|
||||
- output path returned by adapter must match requested output path.
|
||||
- an empty adapter result path means the requested path; adapters cannot select
|
||||
an alternate destination.
|
||||
- candidate collection construction, the shared
|
||||
[checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary),
|
||||
and installation into live state are serialized by transcribe's mutex.
|
||||
Expensive hashing and HTTP work remain outside that critical section.
|
||||
- a canonical file has no completion authority until its manifest checkpoint
|
||||
succeeds. A save failure preserves the last successfully committed
|
||||
collection for ordinary retry.
|
||||
- each successful output is validated before its checkpoint, and cancellation,
|
||||
incomplete dispatch, or a failed aggregate save cannot be reported as
|
||||
aggregate stage success.
|
||||
- removed audio inputs are omitted from the current collection and aggregate
|
||||
output list; reconciliation does not delete arbitrary transcript files.
|
||||
- merge cannot consume a partial versioned collection: it requires transcribe
|
||||
aggregate success and an exact match between current speaker records and
|
||||
declared aggregate outputs.
|
||||
|
||||
## Resume Evidence
|
||||
|
||||
Each completed speaker record binds the speaker ID, session-relative audio
|
||||
path, streamed audio checksum and size, the Narratio-visible WhisperX service
|
||||
URL and recognition language, and the canonical transcript path, checksum,
|
||||
size, and producer run. Changed audio bytes select that speaker for replacement;
|
||||
a changed URL or language selects every speaker. Either case stales fixed
|
||||
descendants through the normal runner decision. Retry count/delay, concurrency,
|
||||
timeout, credentials, and diagnostic locations are operational and do not
|
||||
change this evidence.
|
||||
|
||||
An ordinary retry reuses every valid speaker checkpoint even when aggregate
|
||||
transcribe status is failed, running, or stale. A succeeded stage is skipped only
|
||||
when its stored speaker set and aggregate output evidence also match the current
|
||||
collection. Otherwise, execution repairs those records without retranscribing
|
||||
reusable speakers and stales dependents through the normal runner path.
|
||||
`--force` first durably replaces
|
||||
the collection with an empty versioned collection, so a later ordinary retry
|
||||
cannot resurrect pre-force results; speakers checkpointed by that forced
|
||||
attempt remain reusable.
|
||||
|
||||
WhisperX models or private service configuration not exposed by Narratio's
|
||||
adapter contract cannot be fingerprinted; use `--force` after changing them.
|
||||
An older aggregate-only transcribe record has no trustworthy audio-to-output
|
||||
binding. When transcribe is next selected, every current speaker is therefore
|
||||
generated once and recorded in the versioned collection; later invocations can
|
||||
reuse it normally.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [WhisperX](../integrations/whisperx.md) owns HTTP, retry, timeout, and
|
||||
cancellation semantics.
|
||||
- [Configuration](../config.md#pipeline) owns concurrency and other
|
||||
operator-selected values.
|
||||
- Implementation and tests: `internal/stage/transcribe.go`,
|
||||
`internal/stage/transcribe_reconciliation.go`,
|
||||
`internal/stage/transcribe_execution_state.go`,
|
||||
`internal/stage/transcribe_resume.go`,
|
||||
`internal/stage/transcribe_test.go`,
|
||||
`internal/stage/transcribe_reconciliation_test.go`, and
|
||||
`internal/stage/transcribe_checkpoint_test.go`
|
||||
57
docs/internal/stage-trim.md
Normal file
57
docs/internal/stage-trim.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Stage: trim
|
||||
|
||||
## Purpose
|
||||
|
||||
Produce a final-trimmed transcript. By default, the stage generates bounds and
|
||||
applies a bounds-driven trim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/final.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.trimmed.json` (or configured trim output path)
|
||||
- when trim enabled: `artifacts/session_bounds.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
When `trim.enabled=true`:
|
||||
- runs Scriptorium bounds artifact generation;
|
||||
- optionally runs render-debug output generation;
|
||||
- validates bounds payload against transcript;
|
||||
- derives keep selector;
|
||||
- either copies unchanged transcript or runs Seriatim trim;
|
||||
- validates trimmed transcript and materializes bounds output.
|
||||
|
||||
When `trim.enabled=false`:
|
||||
- copies normalized transcript to trimmed output.
|
||||
|
||||
## Invariants
|
||||
|
||||
- normalized transcript is required input.
|
||||
- bounds output exists only in enabled trim path.
|
||||
- render-debug output is diagnostic and not a declared stage output.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers enablement, the bounds prompt and
|
||||
profile identifiers, the Scriptorium configuration identity, transcript input
|
||||
name, sticky session variable, bounds and trimmed output identities, and the
|
||||
Seriatim trim operation. Diagnostic bounds rendering, diagnostic output paths,
|
||||
timeouts, executable paths, and optional reports are operational and do not
|
||||
invalidate the canonical trimmed transcript.
|
||||
|
||||
Narratio cannot inspect prompt, profile, or configuration content that
|
||||
Scriptorium or Seriatim privately resolves behind a stable identifier. Force
|
||||
`trim` after changing such private content without changing its identifier.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns bounds generation and
|
||||
debug-render subprocess behavior.
|
||||
- [Seriatim](../integrations/seriatim.md) owns transcript trimming behavior.
|
||||
- [Configuration](../config.md#pipeline) owns trim fields and defaults.
|
||||
- Implementation and tests: `internal/stage/trim.go`,
|
||||
`internal/stage/trim_test.go`,
|
||||
`internal/stage/semantic_contracts_refinement.go`
|
||||
72
docs/internal/storage.md
Normal file
72
docs/internal/storage.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Internal: Storage
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the object-store interface and S3 implementation used by Narratio.
|
||||
Remote key layout and lifecycle belong in [Operations](../operations.md), while
|
||||
operator-selected storage fields and credential mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
## Primary Contract
|
||||
|
||||
`storage.ObjectStore` interface:
|
||||
|
||||
- `List(ctx, prefix)`
|
||||
- `Read(ctx, key)` returns an object body and the generation observed with it
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||
- `Delete(ctx, key)` removes one object and succeeds when it is already absent
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
- callers pass full bucket-relative keys;
|
||||
- storage implementations do not infer campaign/session/run prefixes.
|
||||
|
||||
`ReadObjectBounded` is the shared mechanism for small control objects. It opens
|
||||
one object version, returns the metadata observed with that body, rejects an
|
||||
oversized known size before transfer, and still performs a context-aware
|
||||
limit-plus-one read. It closes the body on every exit. Callers own the policy
|
||||
limit and add the control-object category to errors; this helper is not used for
|
||||
large artifact payloads.
|
||||
|
||||
## Composition
|
||||
|
||||
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
|
||||
resolved configuration. The application loads configured filesystem secrets
|
||||
before calling it. The storage adapter consumes already-resolved values; it does
|
||||
not own discovery, defaults, or configuration validation.
|
||||
|
||||
## S3 Backend Behavior
|
||||
|
||||
- normalizes object keys.
|
||||
- `List` paginates and returns normalized `ObjectInfo`.
|
||||
- A truncated S3 listing must supply a new, non-empty continuation token;
|
||||
otherwise listing fails with bucket and prefix context instead of looping.
|
||||
- `Download` writes local files with parent directory creation.
|
||||
- `Upload` streams local file and returns remote metadata.
|
||||
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||
match or absence precondition directly to the provider request and reports a
|
||||
failed precondition without performing a local check-then-write replacement.
|
||||
- `Delete` maps one normalized key to the provider delete-object operation and
|
||||
treats an absent object as success.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- storage layer is stateless regarding manifest/stage progression.
|
||||
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||
not replace owner-specific size policy.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
- A caller that installs remote content into a canonical workspace path first
|
||||
writes a sibling temporary file, validates the complete object against its
|
||||
owner-specific contract, and only then replaces the destination. Temporary
|
||||
files used solely for remote comparisons are removed by their callers and do
|
||||
not represent installed workspace state.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Contract and S3 adapter: `internal/adapters/storage`
|
||||
- Composition: `internal/app/object_store.go`
|
||||
- Tests: `internal/adapters/storage/*_test.go`,
|
||||
`internal/app/object_store_test.go`
|
||||
103
docs/internal/workspace.md
Normal file
103
docs/internal/workspace.md
Normal file
@@ -0,0 +1,103 @@
|
||||
# Internal: Workspace
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the helpers that construct local session and run paths, coordinate
|
||||
single-writer access, and confine cleanup. The authoritative physical layout and
|
||||
retention workflow belong in [Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Path Ownership
|
||||
|
||||
`internal/artifacts` owns canonical session, run, spool, cache, and
|
||||
previous-cache path construction. `SessionPathsFor` provides the session-scoped
|
||||
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||
consume those helpers instead of rebuilding relative paths.
|
||||
|
||||
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||
explicit cleanup roots and removes only normalized proper descendants through
|
||||
the resulting root handle.
|
||||
|
||||
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||
`EnsureWorkspaceDirectory` reapplies the directory mode after creation so a
|
||||
restrictive umask cannot remove group access, while retaining existing ownership
|
||||
and group. Credential paths are outside this contract; the platform-specific
|
||||
operational requirements are in [Operations](../operations.md#workspace-permissions).
|
||||
|
||||
## Run-Local Stage Layout
|
||||
|
||||
`internal/stage/run_local.go` maps stage outputs and diagnostics into an
|
||||
invocation-scoped layout. Successful outputs are validated and atomically
|
||||
materialized into canonical session paths before stage success. Managed
|
||||
previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
validates and renames the external bundle to the unique immutable Notarius
|
||||
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
Prepared references delivered to Notarius use session-durable paths owned by
|
||||
`internal/artifacts` beneath
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
|
||||
content-addressed snapshots are outside run-local layout, survive ordinary
|
||||
published-run cleanup, and are excluded from publish and restore enumeration.
|
||||
|
||||
## Locking
|
||||
|
||||
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||
operating-system lock held on `.lock` (`ErrLockConflict` on contention). The
|
||||
file retains owner metadata after release or process death; its existence is
|
||||
not evidence that a lock is active. Command and restore flows wait for this
|
||||
lock only while their context remains active, and report a release failure.
|
||||
|
||||
The runner assigns every invocation a run-workspace identity but creates the
|
||||
physical directory only before an executing stage that requires it. Fully
|
||||
reused invocations and cleanup-only invocations therefore do not accumulate
|
||||
empty run directories.
|
||||
|
||||
## Cleanup Semantics
|
||||
|
||||
The canonical cleanup stage:
|
||||
|
||||
- runs after notify only when selected by the invocation range;
|
||||
- requires a succeeded publish record with `uploaded=true`, the published run
|
||||
ID, remote commit key, and current commit-pointer key;
|
||||
- consumes the resolved cleanup policy described in
|
||||
[Configuration](../config.md);
|
||||
- refuses unsafe deletes (root delete, traversal, and out-of-root targets);
|
||||
- treats missing targets as successful, idempotent deletion; and
|
||||
- uses its ordinary failed/succeeded manifest stage record for retry. A later
|
||||
full run can skip reusable upstream stages and retry cleanup without
|
||||
republishing.
|
||||
|
||||
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
[Operations](../operations.md#cleanup).
|
||||
|
||||
## Invariants
|
||||
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
- restore installation is confined to validated relative paths beneath the
|
||||
selected session root.
|
||||
- ordinary workspace paths retain group-writable directory and file modes across
|
||||
nested creation, replacement, and Notarius promotion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle installation: `internal/stage/extract.go`
|
||||
- Workspace modes: `internal/fileops/modes.go`
|
||||
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||
`internal/stage/cleanup.go`, `internal/stage/cleanup_plan.go`, and
|
||||
`internal/app/cleanup_targets.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
761
docs/operations.md
Normal file
761
docs/operations.md
Normal file
@@ -0,0 +1,761 @@
|
||||
# Operations Guide
|
||||
|
||||
Operator workflow for running, recovering, and publishing Narratio sessions.
|
||||
|
||||
For command syntax, see [docs/cli.md](./cli.md). For field-level config, see [docs/config.md](./config.md).
|
||||
|
||||
## Campaign and Session Selection
|
||||
|
||||
Campaign selection priority:
|
||||
|
||||
- `--campaign-file`
|
||||
- `--campaign`
|
||||
- `pipeline.campaigns.default_campaign_id`
|
||||
|
||||
Session source priority:
|
||||
|
||||
- `--session`
|
||||
- local default search paths
|
||||
- remote session object (S3) when local session file is not found and storage is configured
|
||||
|
||||
## Session Initialization
|
||||
|
||||
Use `session init` to generate a concrete session file for local or remote use.
|
||||
|
||||
Local file:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Remote session object:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --remote --force
|
||||
```
|
||||
|
||||
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
|
||||
|
||||
Campaigns must provide stable input files for speakers, autocorrect, glossary,
|
||||
players, and party, and may provide an optional spell-catalog overlay. Session
|
||||
files may override those paths for one session. The `prepare` stage
|
||||
materializes them under `inputs/`; configured consumers use the prepared files,
|
||||
never the original campaign or session source paths. Field definitions and
|
||||
source IDs are in [Configuration](./config.md#notarius-reference-bindings).
|
||||
|
||||
## Standard Session Workflow
|
||||
|
||||
1. Select pipeline/campaign/session config.
|
||||
2. Validate session readiness:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
3. (Optional) inspect stage decisions:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
4. Run the pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
5. Check state:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Run, plan, and status output identify the resolved pipeline profile (or `none`)
|
||||
and effective configuration digest. Status distinguishes the current resolved
|
||||
value from the last value persisted in the session manifest, which helps
|
||||
diagnose profile switches without changing resume authority.
|
||||
|
||||
Before switching an operational profile, compare its effective meaning with the
|
||||
production base through `narratio config diff default <profile>`.
|
||||
The command is read-only and succeeds whether it finds differences or not. Its
|
||||
sorted records describe defaulted, expanded concrete configuration—not source
|
||||
file layout—so it can be used to review model, artifact, and publish changes
|
||||
without creating a session or run. Use `narratio config artifacts` to review the
|
||||
concrete artifact identities, definitions, outputs, and dependencies. Select the
|
||||
same campaign explicitly when profiles could resolve different campaign paths;
|
||||
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
for syntax and record format.
|
||||
|
||||
## Stage Execution and Continuation Behavior
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `render`
|
||||
8. `extract`
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
12. `cleanup`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set; stages with semantic
|
||||
configuration contracts additionally require matching versioned evidence;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- forcing a stage marks succeeded transitive dependents as `stale` before the
|
||||
replacement runs; render and extract are independent siblings; and
|
||||
- an executed failure, changed self-skip, or success that replaces a different
|
||||
effective outcome uses the same fixed dependency relation. A
|
||||
repeated self-skip with the same reason and no outputs is stable and does not
|
||||
perpetually rerun dependent work.
|
||||
|
||||
Every aggregate stage except analyze currently provides semantic-configuration
|
||||
evidence; transcribe additionally reconciles per-speaker completion evidence,
|
||||
while analyze retains its more precise per-artifact fingerprints and validator.
|
||||
Prepare additionally validates current stable/local/S3 source identity and the
|
||||
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
||||
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
||||
its fixed descendants without requiring `--force`. Changing prepare selection
|
||||
semantics likewise reruns all fixed descendants; changing
|
||||
WhisperX language/service identity reuses prepare; changing a Seriatim merge
|
||||
transformation reuses prepare and transcribe; and changing an Audita model
|
||||
reuses prepare, transcribe, and merge while rebuilding transcript refinement.
|
||||
A trim change invalidates both render and extract through the fixed dependency
|
||||
relation, while a render-only change preserves the extract sibling.
|
||||
|
||||
Operational timeouts, retry/concurrency tuning, executable paths,
|
||||
workspace/cache/spool placement, reports, diagnostics, and secret values are
|
||||
excluded. Configuration, models, prompts, modules, or resources loaded
|
||||
privately inside external tools remain unobservable to Narratio. If their
|
||||
contents change behind the same configured identifier, explicitly force the
|
||||
affected stage.
|
||||
|
||||
An explicit self-skip is a durable `skipped` stage outcome that later runs
|
||||
reconsider. It differs from successful no-output execution: disabled `render`
|
||||
and `publish`, and absent or no-executable `analyze`, record `succeeded` with
|
||||
metadata and no outputs. Ordinary later runs reuse those successful results;
|
||||
force the affected stage after enabling or configuring it. Optional artifact
|
||||
inputs are omitted only from the consuming artifact invocation and do not make
|
||||
the stage self-skip.
|
||||
|
||||
Single-stage execution:
|
||||
|
||||
```bash
|
||||
narratio run-stage normalize 2026-04-04 --force
|
||||
```
|
||||
|
||||
Contiguous bounded execution uses inclusive canonical endpoints:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04 --from extract --through analyze --force
|
||||
narratio run 2026-04-04 --from extract --through analyze --force
|
||||
```
|
||||
|
||||
Omitting `--from` selects from `prepare`; omitting `--through` selects through
|
||||
`cleanup`. Force applies only within the selected range. Repeating `--from`,
|
||||
`--through`, or `--force` is rejected instead of resolving by argument order.
|
||||
The plan command uses the same selection contract and prints only the selected
|
||||
range. Planning is read-only: it clones the loaded manifest, models selected
|
||||
stage transitions and invalidation in memory, and invokes resume validation
|
||||
without writing the manifest, creating run directories, materializing files,
|
||||
or invoking pipeline adapters. Analyze detail separates explicit targets,
|
||||
prerequisite rebuilds, scheduled execution, and current reuse. This lets a
|
||||
coarsely stale aggregate analyze stage show zero artifact executions when its
|
||||
selected artifact evidence is still semantically current.
|
||||
|
||||
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
||||
prefix stage must already have a session-manifest status of `succeeded` or
|
||||
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
||||
or interrupted prerequisite without creating a run-local directory or changing
|
||||
session state. Widen `--from` to include that stage, or recover it explicitly
|
||||
before retrying. Excluded prefix stages are not resume-validated or repaired as
|
||||
part of the bounded invocation; selected stages still reject missing, unsafe,
|
||||
or manifest-inconsistent inputs at their owning boundary.
|
||||
|
||||
Stages after `--through` are not prerequisites and are never scheduled by the
|
||||
bounded invocation. A selected forced stage can mark one of those succeeded
|
||||
dependents stale through the fixed invalidation relation, but the dependent
|
||||
does not execute until a later invocation selects it. Production composition
|
||||
likewise initializes only collaborators needed by the selected range and
|
||||
shared session lifecycle. In particular, render does not require Notarius or
|
||||
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
||||
the transcription, Seriatim, Audita, or Notarius adapters.
|
||||
|
||||
### Transcription continuation and replacement
|
||||
|
||||
Transcribe checkpoints each validated speaker independently. If a later
|
||||
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
|
||||
save fails, the aggregate stage is not successful and downstream merge cannot
|
||||
consume the partial set. The last successfully saved speaker records and their
|
||||
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
|
||||
|
||||
After correcting the underlying problem, retry normally:
|
||||
|
||||
```bash
|
||||
narratio run-stage transcribe 2026-04-04
|
||||
```
|
||||
|
||||
Narratio hashes the current prepared audio, validates recorded transcript JSON,
|
||||
checksum, size, and location, and calls WhisperX only for missing or obsolete
|
||||
speakers. A changed audio file replaces only its speaker; a changed WhisperX
|
||||
URL or language makes every current speaker non-reusable. Retry, timeout, and
|
||||
concurrency tuning do not invalidate completed speakers.
|
||||
|
||||
Use force only when every current transcript must be replaced, including after
|
||||
a WhisperX model or private server-side configuration changes behind the same
|
||||
URL and language:
|
||||
|
||||
```bash
|
||||
narratio run-stage transcribe 2026-04-04 --force
|
||||
```
|
||||
|
||||
Force durably clears the reusable collection before sending requests. If that
|
||||
forced attempt fails, a later ordinary retry reuses only replacement speakers
|
||||
checkpointed after the reset; it does not revive pre-force results. Do not edit
|
||||
speaker checksums or copy older JSON into the canonical directory to influence
|
||||
reuse.
|
||||
|
||||
The first current Narratio invocation that selects transcribe for an older
|
||||
aggregate-only manifest retranscribes every current speaker once because the
|
||||
old record cannot prove which audio bytes produced each file. Subsequent runs
|
||||
reuse the new versioned evidence normally. The overall session-manifest schema
|
||||
identifier is unchanged, but older strict-decoding Narratio binaries may reject
|
||||
a manifest after current Narratio writes the additive speaker fields; retain a
|
||||
backup or complete the upgrade before relying on downgrade.
|
||||
|
||||
Restoring a manifest does not implicitly publish or restore every raw speaker
|
||||
file. If a recorded canonical transcript is absent locally, ordinary
|
||||
reconciliation schedules that speaker again. Removing run directories does not
|
||||
remove canonical checkpointed transcripts and does not by itself invalidate
|
||||
them.
|
||||
|
||||
## Artifact Development Workflows
|
||||
|
||||
Choose the narrowest workflow that matches the intended replacement:
|
||||
|
||||
| Intent | Command | Preserved and rebuilt state |
|
||||
| --- | --- | --- |
|
||||
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
|
||||
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
|
||||
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
|
||||
|
||||
All three stop before publication. Inspect the results and run
|
||||
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
|
||||
|
||||
For the extraction-plus-analysis development loop, use:
|
||||
|
||||
```bash
|
||||
narratio regenerate-artifacts 2026-04-04
|
||||
narratio regenerate-artifacts 2026-04-04 --artifacts session_recap,player_handout
|
||||
```
|
||||
|
||||
This command is a transparent expansion to a forced bounded `run` from
|
||||
`extract` through `analyze`. Extraction always rebuilds its complete configured
|
||||
bundle. Analysis rebuilds the selected targets and their required analysis
|
||||
prerequisites, or uses the normal default selection when no artifact names are
|
||||
given. The command does not run publish, notify, or cleanup; delivery and
|
||||
retention remain separate operator actions.
|
||||
|
||||
Inspect current artifact evidence, then publish explicitly when the regenerated
|
||||
set is ready:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
If planning or execution reports stale, missing, failed, or tampered
|
||||
analysis evidence, regenerate the affected target instead of copying an older
|
||||
canonical file into place or editing the manifest. See
|
||||
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
|
||||
|
||||
Analyze checkpoints each completed configured artifact before starting later
|
||||
artifact work. After a partial failure or interruption, retry normally: valid
|
||||
earlier checkpoints are reused and execution resumes with failed, stale, or
|
||||
missing work in dependency order. `--force` is unnecessary unless the operator
|
||||
intends to replace otherwise-current artifacts.
|
||||
|
||||
### Developing With A Notarius Configuration Profile
|
||||
|
||||
A Narratio development overlay can explicitly select a Notarius overlay without
|
||||
requiring the names to match. For example, `profiles/testing.yml` may contain:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
config_profile: downstream-economy
|
||||
```
|
||||
|
||||
Selecting Narratio's `testing` profile then supplies `downstream-economy` only
|
||||
when extraction executes. Switching that explicit selector makes a recorded
|
||||
extraction non-current and reruns it when `extract` is in the selected stage
|
||||
range; the usual dependent invalidation then applies. Selecting a profile does
|
||||
not create a separate workspace or retain a separate current extraction result.
|
||||
|
||||
Narratio cannot observe edits inside Notarius configuration behind an unchanged
|
||||
selector. Force extraction after such an edit when a replacement is intended.
|
||||
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
|
||||
use `analyze` when the existing extraction bundle should be preserved. Field
|
||||
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
|
||||
and the exact downstream invocation belongs in the
|
||||
[Notarius integration contract](./integrations/notarius.md#invocation).
|
||||
|
||||
## Artifact Selection
|
||||
|
||||
For a configured artifact family, selecting its family key expands to every
|
||||
concrete character artifact. Select a concrete generated key to operate on one
|
||||
member only. Manifests and plan output retain the concrete key as the durable
|
||||
identity and include the family and character ID as optional provenance.
|
||||
|
||||
`--artifacts` can be used on `run`, `session plan`, `run-stage`, `analyze`, and
|
||||
`publish`. For a bounded run or plan, the selected range must contain `analyze`
|
||||
or `publish`.
|
||||
|
||||
Selection behavior:
|
||||
|
||||
- validates names against `pipeline.scriptorium.artifacts`;
|
||||
- selects explicit analyze targets and permits their required configured
|
||||
prerequisites to be reused or rebuilt first;
|
||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- never partially selects Notarius lanes.
|
||||
|
||||
## Extraction Workflow
|
||||
|
||||
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||
|
||||
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||
and preserves the complete validated Notarius bundle at:
|
||||
|
||||
- `artifacts/notarius/{narratio_run_id}/`
|
||||
|
||||
The directory is immutable once promoted. Configured lanes become
|
||||
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||
the bundle and `index.json` are retained for audit and resume validation but
|
||||
are not selectable or published implicitly.
|
||||
|
||||
Configured Notarius references resolve only from the current manifest-backed
|
||||
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
||||
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
||||
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
|
||||
content-addressed copies under
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
|
||||
SHA-256 digest identifies the reference bytes, so retries and later invocations
|
||||
reuse the same path while changed bytes select a new path. Older snapshots are
|
||||
retained and are not current artifact evidence. A concurrent refresh of
|
||||
canonical prepared files therefore cannot change the bytes consumed by an
|
||||
in-flight invocation.
|
||||
For a canonical party, preparation retains the validated authored party bytes
|
||||
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
||||
The manifest records their checksums separately, with the players input marked
|
||||
as derived from the party; refresh preparation after changing the roster rather
|
||||
than editing either prepared file.
|
||||
Inspect the effective stable-input inventory and
|
||||
prepared-file readiness with:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Reference metadata records selector, source ID, session-relative path,
|
||||
checksum, and byte size, but never payload contents. Changing a prepared
|
||||
reference changes extraction identity: ordinary continuation rejects the old
|
||||
result, reruns Notarius, and marks successful downstream stages stale. If the
|
||||
prepared file is missing or inconsistent with its manifest checksum, repair
|
||||
the source configuration and refresh prepared state first:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier immutable bundle
|
||||
remains available for inspection, but downstream resolution requires a new
|
||||
current successful extraction record.
|
||||
|
||||
Notarius extraction validates its run-local bundle and moves it once to the
|
||||
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||
no duplicate promotion tree is created.
|
||||
|
||||
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
|
||||
inside an executed Notarius invocation. Ordinary extraction forwards their
|
||||
configured values. Forced extraction always disables checkpoint loading and
|
||||
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
|
||||
Refresh regenerates and replaces only the active input's accepted chunk plan;
|
||||
it is not cache deletion.
|
||||
|
||||
Notarius checkpoint recording must be enabled in the deployed Notarius
|
||||
configuration. Narratio neither creates a Notarius pipeline/configuration nor
|
||||
reads it to infer checkpoint availability. Configure explicit persistent
|
||||
Notarius chunk-plan and checkpoint roots when reuse must survive separate
|
||||
Narratio subprocesses. A changed reference, transcript, selected profile,
|
||||
profile content, endpoint, or other Notarius checkpoint identity input can
|
||||
produce a legitimate cold miss.
|
||||
|
||||
There are two independent resume boundaries: Narratio may skip a completed
|
||||
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
|
||||
reuse compatible accepted work only after Narratio has decided extraction must
|
||||
execute. Deleting Narratio's local reference snapshots does not invalidate a
|
||||
complete durable extraction; missing snapshots are reconstructed only when a
|
||||
later extraction actually runs.
|
||||
|
||||
## External Command Lifecycle
|
||||
|
||||
When an external command is cancelled or times out, Narratio terminates its
|
||||
owned descendants as well as the command itself. Cancellation first requests
|
||||
termination where the platform supports it, then force terminates after a
|
||||
bounded wait. A command is not considered finished until its leader has been
|
||||
reaped, and descendants that keep standard output or error open cannot keep
|
||||
the invocation blocked. Other operating systems fail closed rather than launch
|
||||
a command without tree ownership.
|
||||
|
||||
Subprocess stdout and stderr diagnostics are separately redacted and capped at
|
||||
8 MiB per invocation. Narratio does not retain configured credential values in
|
||||
these logs or their error tails; reaching a capture limit terminates the command
|
||||
tree and reports which stream exceeded the limit.
|
||||
|
||||
Run-local diagnostics are:
|
||||
|
||||
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run upload is an allowlist derived from the session manifest's declared
|
||||
current-run outputs, logs, and generated configurations, not a workspace scan.
|
||||
References to canonical session files and prior-run paths are excluded. Each
|
||||
declared source is opened without following symlinked ancestors or the leaf,
|
||||
verified as a regular file, and streamed from that verified descriptor.
|
||||
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
||||
current-run diagnostics uploads no run-local files. No distinguished diagnostic
|
||||
file is uploaded. The durable bundle is never scanned for implicit publication;
|
||||
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio automatically reruns extraction when its recorded invocation contract,
|
||||
prepared Narratio reference identities, or durable output validation changes.
|
||||
The semantic portion covers Notarius enablement, pipeline identity, declared
|
||||
reference mapping, output contracts, and the explicit Notarius configuration
|
||||
profile selector. Executable, timeout, working directory, and private
|
||||
config-file paths are operational and do not invalidate a current result.
|
||||
It cannot fingerprint configuration or profile contents, prompts, modules, or
|
||||
other references loaded transitively by Notarius itself. Force extraction after
|
||||
changing any of those inputs behind an unchanged selector, even when the
|
||||
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||
marks successful downstream stages stale. Ordinary extraction failures or
|
||||
outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
Publish reuse additionally tracks enabled/run-upload behavior, normalized
|
||||
output rules, and remote backend/bucket/region/endpoint/root identity.
|
||||
Credential environment names, local workspace placement, and run IDs are
|
||||
excluded. Regardless of semantic reuse evidence, executing publish reads
|
||||
independent remote locks directly and rechecks initially unlocked sources
|
||||
before commit selection.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
Run publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
Equivalent:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
This standalone command stops after publish and does not invoke cleanup. A
|
||||
default full `run` continues through notify and cleanup.
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
audio and the run-local Notarius staging bundle;
|
||||
- uploads configured published outputs and `previous/**` cache files into the
|
||||
same immutable run scope, including only explicitly configured extraction
|
||||
lanes;
|
||||
- writes `{session_prefix}/runs/{run_id}/commit.json` after all declared
|
||||
immutable objects are uploaded and verified; and
|
||||
- writes `{session_prefix}/current/commit-pointer.json` once, last.
|
||||
|
||||
`current/commit-pointer.json` is the remote current-state commit marker. It
|
||||
selects exactly one immutable commit, which declares the complete object set.
|
||||
|
||||
## Remote Commit Migration
|
||||
|
||||
The immutable remote commit contract uses
|
||||
`runs/{run_id}/commit.json` to declare a run's complete object set and a small
|
||||
`current/commit-pointer.json` to select it. The pointer binds the selected
|
||||
commit by version, checksum, size, and storage generation; committed artifacts
|
||||
are also checksum- and generation-bound. Readers accept this contract now and
|
||||
strictly reject mismatched or unknown data.
|
||||
|
||||
A selected commit may carry a `published_output` declaration from an earlier
|
||||
run in the same campaign/session. Restore, previous-session caching, status,
|
||||
and artifact inspection use that declaration's exact immutable key and stable
|
||||
source; they do not reconstruct a current-run key or scan the session prefix.
|
||||
Other artifact types remain confined to the selected commit's own run.
|
||||
|
||||
When the pointer is absent, remote state is not published; republish the
|
||||
session to establish a committed snapshot. Remote commit state does not carry
|
||||
local `current_pointer_written` metadata.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
Each mutable lock is an independent strict JSON object at
|
||||
`{session_prefix}/locks/{sha256(source)}.lock`. The lowercase digest is computed
|
||||
from the exact canonical source. Listing validates every object in that
|
||||
reserved prefix and reports locks sorted by source.
|
||||
|
||||
Upgrades do not read, migrate, or delete the retired `{session_prefix}/locks.yml`
|
||||
object. It has no effect on publication and may be removed manually after
|
||||
verifying lock commands and publication behavior with the upgraded Narratio
|
||||
version. No migration command is required.
|
||||
|
||||
Operational rules:
|
||||
|
||||
- add uses conditional creation and reports an existing lock;
|
||||
- add with `--force` observes the current generation and attempts one
|
||||
generation-matched replacement. A concurrent change is reported for an
|
||||
explicit operator retry;
|
||||
- remove is idempotent and reports whether a lock was present;
|
||||
- the initial publish lock snapshot controls output planning. A locked source
|
||||
carries its exact declaration from the prior current commit when available;
|
||||
a never-published locked source remains absent even when required;
|
||||
- carried bytes are not copied. Publish verifies the prior object's exact key,
|
||||
size, and generation without downloading it; and
|
||||
- after immutable uploads and the immutable commit, publish re-reads every
|
||||
initially unlocked participating source. A newly added lock prevents the
|
||||
current pointer from advancing, although the immutable attempt may remain.
|
||||
A removed initial lock may conservatively cause one carry.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
narratio session locks 2026-04-04
|
||||
narratio session locks add 2026-04-04 narratio.artifact.session_recap --reason "manual edits" --force
|
||||
narratio session locks remove 2026-04-04 narratio.artifact.session_recap
|
||||
```
|
||||
|
||||
## Restore Workflow
|
||||
|
||||
Use restore when local durable session state is missing or stale and remote committed current state is authoritative.
|
||||
|
||||
Dry run:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Apply:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
`--dry-run` does not write durable session files. It still reads the selected
|
||||
remote current state and may read object identity/content needed to classify the
|
||||
plan, so it is not a network-free operation.
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- the committed session manifest and the committed transcript/artifact objects
|
||||
declared by the selected remote commit
|
||||
- `previous/**` when needed by configured previous-session artifact inputs;
|
||||
each is selected from its exact committed `published_output` declaration
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||
directories, symlinks, or other non-regular local targets
|
||||
|
||||
Restore treats a local leaf symlink or any symlinked ancestor beneath the
|
||||
selected session root as a conflict, even when the linked content matches the
|
||||
selected remote object. `--force` does not override that conflict.
|
||||
|
||||
Before each install, restore validates the destination again beneath the
|
||||
selected session root. A target changed into a symlink, directory, or other
|
||||
non-regular entry is not overwritten.
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
If restore fails after beginning installation, it leaves a durable
|
||||
`.restore-incomplete.json` marker in the session root. Pipeline runs will stop
|
||||
until you rerun the same restore command and it completes. Restore intentionally
|
||||
does not try to roll back files already installed; retrying the selected remote
|
||||
snapshot is the recovery procedure.
|
||||
|
||||
## Local State Layout
|
||||
|
||||
Session root:
|
||||
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
|
||||
Durable session paths:
|
||||
|
||||
- `manifest.json`
|
||||
- `inputs/**`
|
||||
- `audio/**`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**`
|
||||
- `reports/**`
|
||||
- `logs/**`
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
- `cache/notarius/references/<sha256>/<prepared-basename>`
|
||||
|
||||
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||
add missing markers. Move it aside and regenerate it, restore a verified
|
||||
current committed snapshot, or republish healthy regenerated state as the
|
||||
relevant workflow requires.
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
Stable reference snapshots live in the session cache path above. Ordinary
|
||||
published-run cleanup removes its selected run-local target but retains this
|
||||
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
|
||||
or digest directory manually only while no command is active for the session;
|
||||
regular missing or corrupt content is reconstructed on the next executed
|
||||
extraction, while symlinks, directories, and other unsafe conflicts must be
|
||||
investigated and removed deliberately.
|
||||
|
||||
Run-local layout:
|
||||
|
||||
- `runs/{run_id}/{stage}/outputs`
|
||||
- `runs/{run_id}/{stage}/logs`
|
||||
- `runs/{run_id}/{stage}/reports`
|
||||
- `runs/{run_id}/{stage}/config`
|
||||
- `runs/{run_id}/{stage}/scratch`
|
||||
|
||||
Spool layout (runtime/transient):
|
||||
|
||||
- `{spool.root}/{campaign}/{session_id}/{run_id}/...`
|
||||
- restore audio spool under `{spool.root}/{campaign}/{session_id}/restore/audio`
|
||||
|
||||
Cache layout (durable S3 audio cache):
|
||||
|
||||
- `{cache.root}/s3/{bucket}/...`
|
||||
|
||||
Each cached audio file has an adjacent managed identity record. It binds the
|
||||
file to its remote object version and verified digest; deleting or altering the
|
||||
record simply causes Narratio to download and verify the object again.
|
||||
Remote audio and restored files are fully transferred into sibling temporary
|
||||
files and validated before they replace their canonical workspace paths.
|
||||
|
||||
### Workspace Permissions
|
||||
|
||||
Ordinary Narratio workspace content is intentionally shareable with the
|
||||
workspace group. On POSIX systems, Narratio-created workspace, spool, and cache
|
||||
directories converge on setgid `02775`; ordinary files, including manifests,
|
||||
transcripts, generated configuration, logs, reports, and Notarius artifacts,
|
||||
converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
||||
caller umask does not remove group write or setgid. It does not change file or
|
||||
directory ownership: the configured workspace's existing group is inherited.
|
||||
|
||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||
shared workspace or in a separately restricted credential location; ordinary
|
||||
workspace group access must never be treated as authorization to read keys.
|
||||
Provision a credential directory as `0700` and credential files as `0600`;
|
||||
these permission modes are an operator requirement and are not enforced by
|
||||
ConfigKit.
|
||||
|
||||
Every pipeline-loading command bootstraps credentials once, before the first
|
||||
configuration load and before any adapter is constructed. `--secrets-dir`
|
||||
selects an explicit directory (relative paths use caller CWD). Otherwise a
|
||||
present `secrets/` beside the resolved root pipeline is used, while an absent
|
||||
conventional directory is skipped. Explicit missing/empty selections and
|
||||
invalid existing directories fail the command. ConfigKit follows a symlink used
|
||||
as the selected directory, but immediate entries must be regular non-symlink
|
||||
files. Existing environment variables—including empty values—always win.
|
||||
|
||||
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
|
||||
environment references. When both are absent, the AWS SDK retains its default
|
||||
credential provider chain. When both are configured, Narratio supplies their
|
||||
load-time resolved values through an explicit static provider. Audita's optional
|
||||
`llm_api_key` is likewise captured at pipeline load and supplied only through
|
||||
the child process's `AUDITA_LLM_API_KEY` override.
|
||||
|
||||
External adapter results are individually bounded before Narratio validates or
|
||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||
disk exhaustion with filesystem, service, container, or volume quotas sized for
|
||||
the session workload.
|
||||
|
||||
## Cleanup
|
||||
|
||||
Session-scoped cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04
|
||||
```
|
||||
|
||||
Global cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean --all
|
||||
```
|
||||
|
||||
Dry-run and cache variants:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04 --dry-run --clear-cache
|
||||
narratio clean --all --dry-run --clear-cache
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||
root; root, traversal, and out-of-root targets are refused;
|
||||
- the final cleanup stage is gated by successful publish evidence plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
- the manifest publish record has `uploaded=true`, the published run ID, the
|
||||
exact remote commit key, and the current commit-pointer key.
|
||||
|
||||
Cleanup derives only the configured run-scoped spool and workspace targets,
|
||||
uses root-confined deletion, and does not read or mutate object storage. A
|
||||
failure leaves publish succeeded and the remote committed snapshot current.
|
||||
Retry with a normal full run: valid upstream work, including publish and notify,
|
||||
is skipped while the failed cleanup stage runs again. `run-stage cleanup` is
|
||||
also available when the manifest has valid publish evidence. A range ending at
|
||||
`notify` or earlier and the standalone `publish` command never delete local
|
||||
state.
|
||||
|
||||
Older local manifests that contain the retired cleanup-obligation field are
|
||||
intentionally incompatible with strict manifest decoding. If such an
|
||||
obligation is still outstanding, complete it with the pre-upgrade binary.
|
||||
Otherwise verify that the remote publication is committed, move aside or
|
||||
regenerate the incompatible local manifest (or restore the verified remote
|
||||
snapshot), and then use the cleanup stage. Manual JSON editing is not the
|
||||
normal migration path.
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
- Local and S3 audio modes are mutually exclusive.
|
||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||
- Restore requires configured object storage and committed remote current state.
|
||||
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
|
||||
resolution and object-store initialization.
|
||||
305
docs/policy/architecture.md
Normal file
305
docs/policy/architecture.md
Normal file
@@ -0,0 +1,305 @@
|
||||
# Architecture
|
||||
|
||||
This document defines Narratio's intended high-level architecture and the
|
||||
invariants that changes must preserve. Implemented component details belong in
|
||||
the [Internal Overview](../internal/overview.md) and its linked documents.
|
||||
Significant architectural decision history belongs under `docs/adr/` when such
|
||||
records exist.
|
||||
|
||||
## System Shape
|
||||
|
||||
Narratio is a small Go application that turns D&D session audio into polished
|
||||
transcripts and generated session artifacts. It is an explicit, stage-driven
|
||||
orchestrator, not a general workflow engine.
|
||||
|
||||
Narratio coordinates specialized external systems rather than reimplementing
|
||||
their domains:
|
||||
|
||||
- WhisperX performs transcription;
|
||||
- Seriatim performs deterministic transcript processing and rendering;
|
||||
- Audita performs transcript correction and polishing;
|
||||
- Notarius extracts validated structured artifact bundles; and
|
||||
- Scriptorium executes prompts and produces configured artifacts.
|
||||
|
||||
Narratio owns orchestration, configuration resolution, session and run state,
|
||||
artifact and path modeling, manifest persistence, stage sequencing, resume,
|
||||
restore, cleanup gates, and publish semantics. External contracts are defined
|
||||
in the [integration documentation](../integrations/).
|
||||
|
||||
The pipeline has one canonical ordered stage set. Configuration may enable,
|
||||
disable, or parameterize supported behavior, but it must not turn that sequence
|
||||
into an arbitrary DAG or hide orchestration in generic workflow abstractions.
|
||||
An invocation selects either the full sequence or one inclusive contiguous
|
||||
range of it. Execution remains flat and canonical even though invalidation is
|
||||
dependency-aware: the application owns a separate fixed relation used only to
|
||||
stale transitive dependents, including dependents outside a selected range.
|
||||
The implemented stage inventory belongs in the
|
||||
[Internal Overview](../internal/overview.md).
|
||||
|
||||
Narratio is contract-first without being abstraction-heavy. Interfaces and
|
||||
extension points should protect demonstrated boundaries. New abstraction is not
|
||||
itself an architectural goal.
|
||||
|
||||
## Ownership And Dependency Direction
|
||||
|
||||
The application boundary owns command dispatch, configuration selection,
|
||||
production composition, session locking, and top-level lifecycle. It may depend
|
||||
on concrete implementations to assemble a run.
|
||||
|
||||
Stage orchestration expresses intent in Narratio-level data and interfaces.
|
||||
Stages may depend on configuration, manifest, artifact, path, and adapter
|
||||
contracts, but they must not depend on transport-specific request types,
|
||||
subprocess argument construction, cloud SDK types, or downstream tool internals.
|
||||
|
||||
Adapters translate between Narratio contracts and external systems. They own
|
||||
HTTP, subprocess, notification, and object-storage mechanics, including command
|
||||
construction, transport behavior, provider response handling, and external
|
||||
error adaptation. External dependency types must remain inside the adapter that
|
||||
owns them unless that dependency is the adapter's explicit public contract.
|
||||
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||
construction, notification transport, and object-storage SDK details remain
|
||||
behind these boundaries.
|
||||
|
||||
State and path services must not infer stage policy. Storage implementations
|
||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||
or root-prefix semantics. Manifest persistence records transitions but does not
|
||||
choose orchestration policy. Artifact resolution identifies and validates
|
||||
artifacts but does not execute producers.
|
||||
|
||||
Dependencies should remain narrow and point toward Narratio-owned contracts.
|
||||
Prefer the Go standard library. Add an external dependency only when it provides
|
||||
a clear correctness, security, interoperability, or complexity benefit, and
|
||||
confine it to the boundary that needs it.
|
||||
|
||||
## Stage Boundaries
|
||||
|
||||
Each stage has one explicit responsibility and declares:
|
||||
|
||||
- required input state;
|
||||
- produced output state;
|
||||
- configuration it consumes;
|
||||
- external adapters it uses;
|
||||
- manifest references and metadata it reads or writes;
|
||||
- skip, force, invalidation, and resume behavior; and
|
||||
- failure behavior.
|
||||
|
||||
Stages write and validate run-local results before materializing canonical
|
||||
outputs where that distinction applies. A stage is complete only after its
|
||||
required outputs have been written, validated, and recorded in durable manifest
|
||||
state. Later stages depend on recorded success and artifact resolution, not
|
||||
merely on incidental files existing on disk.
|
||||
|
||||
A failed or interrupted stage must not be presented as successful. Failure
|
||||
should preserve enough local state and diagnostics for inspection, recovery,
|
||||
and resume. Forcing a stage invalidates succeeded transitive dependents
|
||||
according to a fixed application-owned relation that is separate from canonical
|
||||
execution order. The relation is validated against the stage inventory and is
|
||||
not configurable.
|
||||
|
||||
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||
outcome is persisted, clears older outputs owned by the stage, and is
|
||||
reconsidered on a later invocation. A stage may also validate whether an
|
||||
otherwise successful recorded result is still resumable; an obsolete result
|
||||
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||
stops execution.
|
||||
|
||||
Shared behavior should live behind a narrow service or helper with one clear
|
||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||
artifact, or path policy ad hoc.
|
||||
|
||||
## Manifest, Resume, And Restore
|
||||
|
||||
The session manifest is the durable ledger for progress across invocations. It
|
||||
records session and run identity, stage state, input and output references,
|
||||
diagnostic references, checksums or provenance where useful, and non-secret
|
||||
adapter and publish metadata.
|
||||
|
||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||
inspected and validated, but file presence alone does not replace recorded
|
||||
stage state. Each invocation retains an in-memory command summary and a
|
||||
run-local workspace identity, but neither is a durable progress ledger. The
|
||||
physical run directory is created on demand only when an executing stage
|
||||
requires it; a fully reused invocation or a cleanup-only invocation does not
|
||||
leave an empty run directory.
|
||||
|
||||
Restore treats committed remote current state as its authority. It must plan
|
||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||
install the validated session manifest after other restored durable files. The
|
||||
physical workflow and recovery procedures belong in
|
||||
[Operations](../operations.md).
|
||||
|
||||
Restore and runner transitions for one session use the same local lock. A
|
||||
durable incomplete-restore marker blocks runner reuse after a partial restore;
|
||||
safe retry, rather than rollback of arbitrary local effects, is the recovery
|
||||
mechanism. Restored manifest-local references must be confined to the selected
|
||||
local session root, never trusted as producer-machine absolute paths.
|
||||
|
||||
For the immutable remote-commit protocol, a restore or status operation binds
|
||||
to one pointer-selected commit and only its declared object identities. A force
|
||||
flag may replace an eligible regular managed file, but never turns a directory
|
||||
or other non-regular conflict into a successful restore.
|
||||
|
||||
## Configuration
|
||||
|
||||
Configuration is strict, explicit, centralized, and operator-oriented.
|
||||
|
||||
- YAML decoding rejects unknown fields.
|
||||
- Defaults are centralized and testable.
|
||||
- Empty configured values do not silently replace meaningful defaults.
|
||||
- Validation rejects invalid configuration before stage execution where
|
||||
practical.
|
||||
- Root-owned imports and one selected profile resolve deterministically through
|
||||
the configuration owner; commands do not implement their own merge rules.
|
||||
- Canonical party rosters are campaign-owned. Their derived players projection
|
||||
and concrete character-family artifacts are resolved before runtime stages
|
||||
or adapters receive configuration.
|
||||
- Resume uses stage- or artifact-owned semantic evidence for observable
|
||||
result-affecting configuration; profile identity and an effective digest are
|
||||
provenance, never blanket cache keys.
|
||||
- Session templating remains narrow and deterministic rather than becoming a
|
||||
general configuration language.
|
||||
- Secret values are supplied indirectly and are not persisted in ordinary
|
||||
configuration.
|
||||
|
||||
Narratio must not become a second configuration system for downstream tools.
|
||||
External systems own their runtime defaults wherever practical; Narratio passes
|
||||
the paths required by its stage contracts and explicit operator overrides. The
|
||||
field-level contract and credential-supply mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
## Artifacts, Paths, And Storage
|
||||
|
||||
Artifact identities and local and remote paths are application contracts.
|
||||
Canonical helpers own workspace, spool, cache, session, run, input, transcript,
|
||||
artifact, log, report, configuration, and publish-current paths. Callers must
|
||||
not reconstruct canonical paths through scattered string concatenation.
|
||||
|
||||
Reusable audio cache entries require a typed record that binds a confined,
|
||||
no-follow regular file and its digest to the selected remote object identity.
|
||||
Size alone and unqualified multipart ETags are not content-integrity evidence.
|
||||
|
||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||
canonical outputs before reporting success, and consumers resolve declared
|
||||
artifact identities rather than infer files from unrelated directory contents.
|
||||
External artifact bundles become current only through validated immutable
|
||||
promotion and manifest records; directory presence alone never establishes
|
||||
availability.
|
||||
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
relative destinations must not expand the scope of an operation. Cleanup is
|
||||
permitted only through explicit operator action or the configured final cleanup
|
||||
stage, and it must preserve durable cache unless cache removal is explicitly
|
||||
requested.
|
||||
|
||||
Physical layout, retention, and operational lifecycle belong in
|
||||
[Operations](../operations.md). Logical external formats and durable integration
|
||||
contracts belong under [Integrations](../integrations/).
|
||||
|
||||
## Publish Commit Boundary
|
||||
|
||||
Publish has one explicit remote commit boundary. A remote run becomes current
|
||||
only after Narratio has successfully uploaded its immutable run-scoped objects,
|
||||
the immutable commit manifest, and finally the current commit pointer.
|
||||
|
||||
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||
must not be presented as current remote state. Publish locks remain authoritative
|
||||
and are not bypassed by a forced run. Each mutable remote lock is an independent
|
||||
per-source object; lock mutations use provider-enforced generation preconditions.
|
||||
Publish revalidates every initially unlocked participating source after writing
|
||||
the immutable commit and before pointer selection. A lock observed then leaves
|
||||
the prior committed snapshot current. A lock created after that final check may
|
||||
race the pointer write; the commit protocol does not claim a stronger distributed
|
||||
transaction boundary.
|
||||
|
||||
Automatic local cleanup is permitted only after a successful publish commit,
|
||||
only when explicitly configured, and only through the path-safety guardrails.
|
||||
It is the final ordinary pipeline stage after `notify`. The stage derives its
|
||||
exact run-scoped targets from validated publish metadata and the configured
|
||||
campaign/session identity; it never infers a target from directory contents or
|
||||
rereads remote storage. A cleanup failure makes the invocation incomplete while
|
||||
leaving publish succeeded and the committed remote snapshot authoritative. A
|
||||
normal later full run reuses valid upstream stages and retries cleanup.
|
||||
|
||||
The standalone `publish` command selects only publish and therefore never
|
||||
invokes cleanup. Operators may select cleanup directly only when the manifest
|
||||
contains valid successful publication evidence.
|
||||
|
||||
## Security, Privacy, And Diagnostics
|
||||
|
||||
Narratio distinguishes ordinary workspace data from credentials. Campaign and
|
||||
session material—including manifests, transcripts, prompts, generated
|
||||
configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
||||
intentionally shareable with the configured workspace group. API-key material
|
||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||
|
||||
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
|
||||
a cross-build gate and release asset, but no native runtime release gate.
|
||||
Windows and macOS amd64 are unsupported and have no compatibility promise or
|
||||
release asset.
|
||||
|
||||
On supported systems, Narratio-created ordinary workspace directories converge on
|
||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||
umask is restrictive. This preserves the existing workspace group for nested
|
||||
creation and atomic replacements without changing ownership. API-key storage
|
||||
uses a separate restrictive contract.
|
||||
|
||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||
to manifests, logs, generated configuration, reports, publish metadata,
|
||||
documentation, or examples. Pipeline configuration names environment variables;
|
||||
command-owned startup may populate previously unset variables from ConfigKit's
|
||||
explicit or conventional secrets directory. This bootstrap occurs once per
|
||||
invocation before any typed configuration load or adapter construction and is
|
||||
not hidden inside library loaders. Safe bootstrap diagnostics may identify
|
||||
files and statuses but never values. Diagnostics should avoid transcript and
|
||||
prompt content unless a deliberate, bounded inspection mechanism requires it.
|
||||
|
||||
Logs, reports, generated invocation files, generated configuration, and render
|
||||
debug files are diagnostics, not canonical pipeline products. They should be
|
||||
durable and discoverable where configured, and manifest references must preserve
|
||||
the distinction between diagnostics and artifacts.
|
||||
|
||||
Documentation security rules belong in the
|
||||
[Documentation Policy](documentation.md). Credential supply belongs in
|
||||
[Configuration](../config.md), while permissions, sensitive runtime-artifact
|
||||
handling, and recovery belong in [Operations](../operations.md).
|
||||
|
||||
## Determinism And Testability
|
||||
|
||||
Narratio prefers deterministic behavior where practical, including stable local
|
||||
and remote layouts, sorted operation order, predictable generated
|
||||
configuration, repeatable command construction, deterministic artifact
|
||||
resolution, and reproducible planning.
|
||||
|
||||
Run IDs and timestamps may be intentionally variable, but surrounding behavior
|
||||
must remain controllable in tests. Core behavior should be testable without live
|
||||
external services; expensive, nondeterministic, destructive, or external
|
||||
boundaries should be replaceable with focused test doubles. General testing
|
||||
philosophy and sufficiency rules belong in the [Testing Policy](testing.md).
|
||||
|
||||
## Documentation And Decision Records
|
||||
|
||||
Documentation follows the [Documentation Policy](documentation.md). Current
|
||||
behavior belongs in its canonical user, operator, integration, architecture, or
|
||||
internal owner. Proposed behavior and implementation status belong under
|
||||
`docs/roadmap/`.
|
||||
|
||||
Significant architectural decisions may be recorded under `docs/adr/` using the
|
||||
format and lifecycle defined by the documentation policy. ADR acceptance does
|
||||
not establish that a decision has been implemented.
|
||||
|
||||
## Architectural Non-Goals
|
||||
|
||||
Narratio does not aim to provide:
|
||||
|
||||
- a generic DAG or workflow engine;
|
||||
- a replacement configuration layer for WhisperX, Seriatim, Audita,
|
||||
Scriptorium, or other downstream tools;
|
||||
- a storage abstraction broader than the needs of this pipeline;
|
||||
- stage logic coupled directly to cloud SDKs, transports, subprocess details,
|
||||
or downstream implementation internals;
|
||||
- raw-secret persistence;
|
||||
- implicit cross-stage behavior that bypasses manifest and artifact contracts;
|
||||
or
|
||||
- a prompt-authoring system.
|
||||
150
docs/policy/documentation.md
Normal file
150
docs/policy/documentation.md
Normal file
@@ -0,0 +1,150 @@
|
||||
# Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep Narratio documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
stage or integration keys, schemas, file names, paths, status codes, retry
|
||||
behavior, and runtime guarantees. If readers could reasonably treat a statement
|
||||
as a contract, maintain it only in the owning document.
|
||||
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Maintainer release procedure | `docs/release.md` | Version selection, candidate preparation and validation, guarded tag publication, release completion boundary, failure recovery, and optional asynchronous inspection. | Script implementation mechanics, current application contracts, and historical release summaries. |
|
||||
| Historical release summary | `docs/releases/<tag>.md` | Immutable summary, compatibility, upgrade, and changes for one released version. | Current maintainer procedure and current application contract details. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, output conventions, and exit behavior. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, stage implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schemas, fields, defaults, environment overrides, validation rules, and user-selectable stage or integration settings. | Complete example files, CLI syntax, runtime state lifecycle, implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and remote-state layout, output and diagnostic handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical artifact schemas, implementation mechanics. |
|
||||
| Troubleshooting | `docs/troubleshooting.md` | Symptom-driven diagnosis, likely causes, safe inspection steps and remedies, and links to relevant contracts. | CLI syntax, configuration definitions, operational procedures, integration contracts, implementation mechanics. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical artifact paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, Operations, And Troubleshooting
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. Troubleshooting
|
||||
starts from observable symptoms and links readers to the owning command,
|
||||
configuration, operational, or integration contract. When a workflow crosses
|
||||
these topics, choose the document that owns the task and link to the other
|
||||
contracts.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how Narratio implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Troubleshooting owns safe diagnostic and remediation guidance.
|
||||
Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
296
docs/policy/testing.md
Normal file
296
docs/policy/testing.md
Normal file
@@ -0,0 +1,296 @@
|
||||
# Testing Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
104
docs/release.md
Normal file
104
docs/release.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# Releasing Narratio
|
||||
|
||||
This document is the maintainer procedure for creating a Narratio source and
|
||||
binary release. The synchronous release boundary is a successful push of one
|
||||
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
|
||||
change that result.
|
||||
|
||||
## Go prerequisite
|
||||
|
||||
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
|
||||
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
|
||||
asset construction; that asynchronous work does not change the tag-push
|
||||
completion boundary.
|
||||
|
||||
## Choose a version and write its note
|
||||
|
||||
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
|
||||
`vMAJOR.MINOR.PATCH`:
|
||||
|
||||
- increment `MINOR` for backward-compatible features;
|
||||
- increment `PATCH` for backward-compatible fixes; and
|
||||
- reserve a new `MAJOR` for an intentional breaking documented contract.
|
||||
|
||||
Before preparing the candidate, create
|
||||
`docs/releases/vMAJOR.MINOR.PATCH.md` with this structure:
|
||||
|
||||
```markdown
|
||||
# Narratio vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
The compatibility section identifies relevant CLI, configuration, artifact,
|
||||
integration, or operating-contract changes. The upgrade section states the
|
||||
required operator action, or explicitly says that no special action is
|
||||
required. Release notes are immutable historical summaries; link to the
|
||||
current canonical documentation for detailed behavior.
|
||||
|
||||
Commit the note and all candidate changes, then use the ordinary development
|
||||
workflow to push that commit to `main`. Do not create a release tag before the
|
||||
candidate is committed and `origin/main` contains the exact same commit.
|
||||
|
||||
## Validate the candidate
|
||||
|
||||
Run the shared checker from any directory:
|
||||
|
||||
```sh
|
||||
scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
It validates the version and matching note, module hygiene, formatting,
|
||||
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
||||
and three official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||
application services, CI, or Gitea, and does not create tags or modify tracked
|
||||
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
||||
checker.
|
||||
|
||||
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
|
||||
`arm64`. The macOS cross-build proves compilation; it is not native macOS
|
||||
runtime evidence.
|
||||
|
||||
## Publish the tag
|
||||
|
||||
From a clean checkout on `main` whose `HEAD` equals `origin/main`, run:
|
||||
|
||||
```sh
|
||||
scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The command fetches and checks `origin/main`, re-runs candidate validation,
|
||||
then fetches and checks again before creating an explicitly unsigned lightweight
|
||||
tag for the originally recorded commit. It refuses dirty, divergent, changed,
|
||||
or already-tagged candidates. It pushes only:
|
||||
|
||||
```text
|
||||
refs/tags/vMAJOR.MINOR.PATCH:refs/tags/vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
It never commits changes, pushes `main`, force-pushes, moves a tag, or pushes
|
||||
all tags. A successful push of that exact ref completes the release command;
|
||||
the command prints the tag and commit, then returns without waiting for CI,
|
||||
querying Gitea, downloading assets, or checking checksums.
|
||||
|
||||
If a failure occurs before the tag is created, correct the candidate on `main`
|
||||
and repeat validation. If the push fails after local tag creation, the local tag
|
||||
is intentionally retained for inspection and the command must not be retried
|
||||
blindly. Once the upstream tag has been pushed, it is immutable. Correct any
|
||||
defect or failed asynchronous publication with a new patch version, a new
|
||||
release note, and the complete procedure again.
|
||||
|
||||
## Optional asynchronous inspection
|
||||
|
||||
After a successful tag push, a human may later inspect the tag-triggered
|
||||
Woodpecker run and the corresponding Gitea release for binaries and checksums.
|
||||
This is optional follow-up only. Automated releasers must not wait for, poll,
|
||||
or treat CI/Gitea completion as a condition of the successful tag push.
|
||||
28
docs/releases/README.md
Normal file
28
docs/releases/README.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Release Notes
|
||||
|
||||
This directory contains immutable historical release notes for Narratio.
|
||||
Future notes are created with the matching stable version and use this minimum
|
||||
structure:
|
||||
|
||||
```markdown
|
||||
# Narratio vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
See the [release procedure](../release.md) for creating a candidate and tag.
|
||||
When asynchronous publication succeeds, the corresponding Gitea release is the
|
||||
canonical source for downloadable binaries and checksums.
|
||||
|
||||
- [v2.0.0](v2.0.0.md)
|
||||
- [v1.7.0](v1.7.0.md)
|
||||
- [v1.6.0](v1.6.0.md)
|
||||
- [v1.5.0](v1.5.0.md)
|
||||
47
docs/releases/v1.5.0.md
Normal file
47
docs/releases/v1.5.0.md
Normal file
@@ -0,0 +1,47 @@
|
||||
# Narratio v1.5.0
|
||||
|
||||
Narratio v1.5.0 makes repeated post-transcript artifact development faster and
|
||||
more explicit while retaining the fixed, stage-driven pipeline model.
|
||||
|
||||
## Highlights
|
||||
|
||||
- The canonical pipeline now completes deterministic rendering before
|
||||
extraction, cleanly separating transcript-generating stages from
|
||||
artifact-generating stages.
|
||||
- `narratio run` and `narratio session plan` accept inclusive `--from` and
|
||||
`--through` bounds. Excluded transcript stages are not executed or
|
||||
invalidated by a bounded artifact-regeneration run.
|
||||
- `narratio regenerate-artifacts SESSION` is an exact convenience alias for a
|
||||
forced run from `extract` through `analyze`, including focused
|
||||
`--artifacts` selections.
|
||||
- Configured Scriptorium artifacts now have independent,
|
||||
manifest-authoritative freshness. Narratio reuses validated current work,
|
||||
rebuilds stale prerequisites in dependency order, and persists successful,
|
||||
failed, and newly stale artifact state when an analysis invocation only
|
||||
partially succeeds.
|
||||
- Publish consumes only configured artifacts backed by current manifest
|
||||
evidence; incidental or tampered files are not promoted as current output.
|
||||
|
||||
## Reliability And Administration
|
||||
|
||||
- Bounded prerequisites are checked again under the session lock before any
|
||||
run mutation, closing a concurrent-run race.
|
||||
- Analysis fingerprints are stable across executable and configuration path
|
||||
changes and continue to cover only Narratio-observable semantic inputs.
|
||||
- Runner composition now carries one validated execution plan from command
|
||||
parsing through prerequisite validation, adapter composition, manifest
|
||||
recording, and stage execution.
|
||||
- `narratio version` reports the exact tag embedded in official release
|
||||
binaries; ordinary source builds report `dev`.
|
||||
|
||||
## Upgrade Notes
|
||||
|
||||
- Existing unbounded commands and direct `run-stage`, `analyze`, and `publish`
|
||||
workflows retain their meanings.
|
||||
- Manifests written before artifact-level analysis state remain readable.
|
||||
Legacy aggregate analysis success is not sufficient freshness evidence, so
|
||||
the first analysis evaluation after upgrading may regenerate configured
|
||||
artifacts once.
|
||||
- Narratio cannot observe executable contents or configuration, prompt,
|
||||
profile, module, and other files loaded privately by Scriptorium. Explicitly
|
||||
force affected artifacts after changing those private inputs.
|
||||
74
docs/releases/v1.6.0.md
Normal file
74
docs/releases/v1.6.0.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Narratio v1.6.0
|
||||
|
||||
Narratio v1.6.0 makes large pipeline configurations easier to organize,
|
||||
inspect, and vary while adding character-oriented artifact generation and a
|
||||
guarded, reproducible release procedure.
|
||||
|
||||
## Summary
|
||||
|
||||
Pipeline configuration can now be assembled from explicit additive imports and
|
||||
a selected production or testing profile. Campaigns can own a canonical,
|
||||
versioned party roster, and Scriptorium artifact families can expand one
|
||||
definition into concrete per-character artifacts, dependencies, variables, and
|
||||
publish rules.
|
||||
|
||||
New read-only configuration commands expose the fully resolved pipeline,
|
||||
source provenance, semantic digest, and profile differences before a session is
|
||||
run. Stage reuse now records configuration-sensitive semantic evidence so
|
||||
profile or configuration changes cannot silently reuse incompatible work.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a backward-compatible feature release. Existing monolithic pipeline
|
||||
files, concrete Scriptorium artifacts, publish rules, and configurations
|
||||
without profiles remain supported. When profiles are declared and no explicit
|
||||
profile is selected, Narratio uses the configured production default.
|
||||
|
||||
An unversioned party file plus a separate players file remains available as an
|
||||
isolated legacy compatibility path, but it cannot drive artifact families. New
|
||||
campaigns and new party-oriented features should use the `narratio.party.v1`
|
||||
schema. Existing manifests remain readable; missing legacy semantic evidence
|
||||
is treated as stale rather than trusted.
|
||||
|
||||
Narratio continues to consume the documented Notarius D&D pipeline contract.
|
||||
The release workflow still cross-compiles Linux, macOS, and Windows binaries
|
||||
for `amd64` and `arm64`; cross-compilation is not native runtime evidence for
|
||||
macOS or Windows.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special action is required for existing monolithic configurations that do
|
||||
not adopt profiles or artifact families. On the first run after upgrading,
|
||||
stages recorded by older manifests may regenerate once because those records do
|
||||
not contain the new semantic configuration evidence.
|
||||
|
||||
To adopt the new configuration model, use the maintained
|
||||
`examples/production-testing` bundle as a migration reference: split stable
|
||||
settings into explicit imports, define a production default and optional
|
||||
testing profile, convert campaign party data to `narratio.party.v1`, remove the
|
||||
separate players file, and then introduce character artifact families. Review
|
||||
the result with `narratio config validate`, `config show`, `config sources`, and
|
||||
`config diff` before running a session.
|
||||
|
||||
Campaign, session, previous-session, and run identifiers must satisfy the
|
||||
documented portable identity grammar. Existing manifests or remote state with
|
||||
unsafe legacy identifiers must be migrated before use.
|
||||
|
||||
## Changes
|
||||
|
||||
- Added root-owned, non-recursive additive pipeline imports with strict,
|
||||
source-aware conflict detection.
|
||||
- Added named pipeline profiles with an explicit production default and
|
||||
deliberate command-line selection.
|
||||
- Added `config validate`, `config show`, `config sources`, and `config diff`
|
||||
for read-only inspection of resolved configuration and provenance.
|
||||
- Added the strict `narratio.party.v1` campaign roster, including stable
|
||||
character IDs, player and character names, optional aliases, and classes.
|
||||
- Added deterministic players derivation and canonical party delivery to
|
||||
downstream integrations.
|
||||
- Added character-oriented artifact families, corresponding member
|
||||
dependencies, family selection, and generated publish policies.
|
||||
- Added semantic configuration fingerprints and stage-specific resume checks
|
||||
across transcript and artifact stages.
|
||||
- Added shared release candidate, asset build, and guarded tag-publication
|
||||
scripts, with tag-triggered publication remaining asynchronous.
|
||||
100
docs/releases/v1.7.0.md
Normal file
100
docs/releases/v1.7.0.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Narratio v1.7.0
|
||||
|
||||
Narratio v1.7.0 completes the maintainability refactor begun after v1.6.0. It
|
||||
simplifies filesystem and process handling, makes resumable work more precise,
|
||||
replaces legacy local and remote state with explicit authorities, and turns
|
||||
cleanup into the final ordinary pipeline stage.
|
||||
|
||||
## Summary
|
||||
|
||||
The session manifest is now the sole durable local progress ledger. Analyze
|
||||
checkpoints successful artifacts individually, allowing an interrupted or
|
||||
partially failed analysis to resume at the first incomplete artifact without
|
||||
rerunning successful siblings unless forced.
|
||||
|
||||
Remote publication now commits immutable run state before advancing one
|
||||
verified current pointer. Publish locks are independent per-source objects;
|
||||
locked outputs retain their previously committed declarations rather than
|
||||
being overwritten or silently omitted. Local cleanup runs after notification
|
||||
as a separately recorded, safely retryable stage.
|
||||
|
||||
The release also adopts Go 1.26, simplifies ordinary file operations around
|
||||
Narratio's trusted group-owned workspace model, improves subprocess lifecycle
|
||||
handling, decomposes several large orchestration and validation components,
|
||||
and removes mutable global test seams.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a minor release with deliberate removal of compatibility paths that
|
||||
were deprecated and documented as subject to removal:
|
||||
|
||||
- Campaigns must use one versioned `narratio.party.v1` `party.yml`. Separate
|
||||
players files, session-level party overrides, and unversioned party documents
|
||||
are rejected.
|
||||
- Session manifests must declare `schema_version:
|
||||
narratio.session-manifest.v1` and satisfy the current strict schema. Older
|
||||
unversioned state and manifests containing the retired
|
||||
`post_publish_cleanup` obligation are rejected rather than migrated.
|
||||
- Remote current state must be selected by
|
||||
`current/commit-pointer.json` and its referenced immutable run commit. Legacy
|
||||
current-state layouts are no longer inferred.
|
||||
- The retired aggregate `locks.yml` object is ignored. Publish locks now live
|
||||
as independent strict objects under the session lock prefix.
|
||||
- Run manifests are no longer produced or consumed. The session manifest owns
|
||||
durable stage progress; invocation summaries and run workspaces remain
|
||||
operational data rather than a second progress authority.
|
||||
|
||||
The CLI and current pipeline/configuration formats otherwise remain compatible.
|
||||
Building and maintaining Narratio now requires Go 1.26. Cross-built Linux,
|
||||
macOS, and Windows assets remain available for `amd64` and `arm64`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Before upgrading a campaign that still uses compatibility data:
|
||||
|
||||
1. Replace legacy party/player inputs with the documented
|
||||
[`narratio.party.v1`](../integrations/party.md) campaign roster.
|
||||
2. If a local manifest contains an outstanding `post_publish_cleanup`
|
||||
obligation, complete it with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, or
|
||||
restore a verified committed snapshot. Do not migrate state by manually
|
||||
editing JSON.
|
||||
3. Republish sessions that do not yet have a valid immutable run commit and
|
||||
`current/commit-pointer.json`; legacy remote layouts are not current state in
|
||||
v1.7.0.
|
||||
4. Recreate any required publication locks with `narratio session locks add`.
|
||||
After verifying the new locks and publication behavior, the ignored legacy
|
||||
`locks.yml` object may be removed manually.
|
||||
|
||||
Existing compatible session manifests and already committed remote snapshots
|
||||
need no special action. Consult [Operations](../operations.md) and
|
||||
[Troubleshooting](../troubleshooting.md) for recovery procedures.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded the maintained development and release baseline to Go 1.26.
|
||||
- Reworked subprocess execution and Windows process-tree supervision while
|
||||
preserving contextual wait and cleanup failures.
|
||||
- Replaced custom confined ordinary I/O and directory-promotion machinery with
|
||||
simpler standard-library operations while retaining bounded reads, atomic
|
||||
single-file visibility, workspace modes, validated downloads, and
|
||||
root-confined destructive operations.
|
||||
- Removed legacy party, manifest, artifact-discovery, remote-current, and lock
|
||||
compatibility paths.
|
||||
- Consolidated durable progress in the session manifest and removed the run
|
||||
manifest domain model.
|
||||
- Unified runner planning and execution decisions, separated invocation
|
||||
lifecycle responsibilities, and avoided creating empty run workspaces for
|
||||
fully reused or cleanup-only invocations.
|
||||
- Added per-artifact analyze execution, checkpointing, reconciliation, and
|
||||
recovery.
|
||||
- Decomposed configuration validation by domain and made validation diagnostics
|
||||
deterministic without treating incidental ordering or wording as a contract.
|
||||
- Introduced immutable remote publish commits, one final current pointer,
|
||||
explicit output planning, safe cross-run carried outputs, and independent
|
||||
per-artifact remote locks.
|
||||
- Added `cleanup` as the final canonical stage after `notify`, with validated
|
||||
publish evidence, safe retry, explicit selection, and no separate cleanup
|
||||
obligation.
|
||||
- Replaced mutable command test hooks with invocation-scoped dependencies and
|
||||
improved structural test diagnostics.
|
||||
94
docs/releases/v2.0.0.md
Normal file
94
docs/releases/v2.0.0.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Narratio v2.0.0
|
||||
|
||||
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
|
||||
simplifies the supported release platforms, and makes artifact definitions and
|
||||
their effective dependency graph directly inspectable.
|
||||
|
||||
## Summary
|
||||
|
||||
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
|
||||
imports, fixed-path named profiles, provenance, defaults, and typed credential
|
||||
references. The recommended production layout keeps production values in the
|
||||
base, defines each ordinary artifact and artifact family in its own imported
|
||||
file, and keeps testing changes in `profiles/testing.yml`.
|
||||
|
||||
Configured-artifact inputs now create their dependency edges automatically.
|
||||
`narratio config artifacts` reports the finalized concrete catalog, including
|
||||
generated party-family members, without loading a session or initializing
|
||||
runtime adapters.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a breaking configuration and platform release. There is no runtime
|
||||
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
|
||||
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
|
||||
presence/default behavior. Defaults apply only to absent values; explicit empty,
|
||||
zero, false, and null values retain their typed meaning and may fail Narratio
|
||||
domain validation.
|
||||
|
||||
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
|
||||
and macOS amd64 builds are no longer produced or supported. Existing v1 release
|
||||
notes remain historical descriptions of their releases.
|
||||
|
||||
Inspection commands still avoid session, manifest, workspace, adapter, and
|
||||
remote-service activity, but they are not credential-free loaders: every
|
||||
explicit typed credential reference in the selected effective configuration must
|
||||
resolve; references replaced or removed by that profile need not resolve.
|
||||
Pipelines containing artifact families also require a campaign so the canonical
|
||||
party can be expanded.
|
||||
`config show` is a secret-safe review projection, not a promised reloadable
|
||||
authoring file.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Migrate each operational bundle before running v2:
|
||||
|
||||
1. Replace old composition syntax with root-owned
|
||||
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
|
||||
additive base sources; they do not override one another.
|
||||
2. Fold the old production overlay into the base pipeline. Move testing changes
|
||||
to `profiles/testing.yml`; compare the two independent resolutions with
|
||||
`narratio config diff default testing`.
|
||||
3. Follow the maintained
|
||||
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
|
||||
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
|
||||
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
|
||||
stable artifact identity.
|
||||
4. Move credential files to an untracked `secrets/` directory beside
|
||||
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
|
||||
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
|
||||
5. For static S3 credentials, configure both S3 references (commonly
|
||||
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
|
||||
To use instance roles, workload identity, shared SDK files, or another AWS SDK
|
||||
discovery provider, omit both references. A partial pair is invalid.
|
||||
6. Remove a `depends_on` entry when it only duplicates a
|
||||
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
|
||||
without supplying the upstream output as an input.
|
||||
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
|
||||
that previously acquired defaults after being explicitly empty. Do not use
|
||||
`config show` output as a source migration mechanism.
|
||||
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
|
||||
with the same profile, campaign, and secrets selection intended for runtime.
|
||||
Resolve the campaign-owned party whenever families are present.
|
||||
9. Update deployment and download automation to one of the three maintained
|
||||
platform assets before removing the v1 binary.
|
||||
|
||||
The [configuration reference](../config.md), [CLI reference](../cli.md), and
|
||||
[maintained examples](../../examples/README.md) are the canonical detailed
|
||||
guides.
|
||||
|
||||
## Changes
|
||||
|
||||
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
|
||||
while keeping Narratio domain validation and post-campaign expansion explicit.
|
||||
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
|
||||
unified safe effective show/diff/digest projection, and source provenance.
|
||||
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
|
||||
Audita credential references without exposing resolved values.
|
||||
- Inferred, normalized, and validated configured-artifact input dependencies as
|
||||
part of the one concrete graph consumed by analyze planning and resume logic.
|
||||
- Added `config artifacts` with stable concrete identities, definition sources,
|
||||
effective dependencies, and generated family/party provenance.
|
||||
- Added the maintained production-base/testing-overlay artifact authoring layout.
|
||||
- Reduced release assets and active platform support to Linux amd64/arm64 and
|
||||
macOS arm64.
|
||||
70
docs/releases/v2.1.0.md
Normal file
70
docs/releases/v2.1.0.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# Narratio v2.1.0
|
||||
|
||||
This release adds explicit Notarius credential forwarding and cache reuse,
|
||||
with predictable forced regeneration and clearer artifact-development guidance.
|
||||
|
||||
## Summary
|
||||
|
||||
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
|
||||
its accepted execution checkpoints. Stable, verified reference snapshots allow
|
||||
unchanged references to retain their cache identity across Narratio invocations.
|
||||
Configuration loading now uses ConfigKit v0.3.0.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
|
||||
fields are optional. Existing v2 configuration needs no schema migration.
|
||||
Configured credential references must resolve during configuration loading,
|
||||
even when extraction will not execute; omitted references add no requirement.
|
||||
|
||||
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
|
||||
refresh. This includes `regenerate-artifacts`, which remains an alias for a
|
||||
forced run from extract through analyze. `analyze` continues to force analysis
|
||||
only, preserving extraction results and transcripts.
|
||||
|
||||
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
|
||||
minimum-version gate. The executable must support the flags selected for the
|
||||
invocation, including `--chunk_cache refresh` for forced extraction. Reference
|
||||
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
|
||||
schemas are unchanged; reused producers may report zero production attempts.
|
||||
|
||||
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
|
||||
arm64. macOS validation is cross-compilation, not native runtime testing.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special migration is required for existing v2 configuration. To use the new
|
||||
features:
|
||||
|
||||
- Declare `notarius.llm_api_key` with the environment-variable name expected by
|
||||
the deployed Notarius profile, supplying its value through the existing
|
||||
environment or secrets-directory mechanism.
|
||||
- Enable checkpoint recording in Notarius configuration before setting
|
||||
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
|
||||
roots for service deployments. An optional `notarius.chunk_cache` overrides
|
||||
its ordinary chunk-plan policy.
|
||||
- Expect an initial checkpoint miss when moving from old run-local reference
|
||||
paths to stable session snapshots. No migration or deletion of old caches is
|
||||
required. Force refreshes the current input's plan, not the whole cache.
|
||||
- Use `narratio analyze SESSION` for analysis-only iteration, and
|
||||
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
|
||||
|
||||
See [Configuration](../config.md), the
|
||||
[Notarius integration contract](../integrations/notarius.md), and
|
||||
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
|
||||
for current details.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
|
||||
from machine configuration.
|
||||
- Added optional typed Notarius credentials, exact-name environment delivery,
|
||||
safe configuration projections, and captured-stream redaction for custom names.
|
||||
- Added Notarius resume and chunk-cache controls with force taking precedence.
|
||||
- Added verified, content-addressed session reference snapshots that survive
|
||||
ordinary run cleanup without becoming published artifacts.
|
||||
- Preserved relative-workspace support for snapshot delivery and confinement.
|
||||
- Kept credentials and cache-policy controls outside extract's semantic resume
|
||||
identity, so operational changes alone do not invalidate completed results.
|
||||
- Added an opt-in real Notarius integration test using a local provider to
|
||||
verify reuse, partial-failure recovery, and forced refresh without paid calls.
|
||||
33
docs/releases/v2.1.1.md
Normal file
33
docs/releases/v2.1.1.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Narratio v2.1.1
|
||||
|
||||
This release fixes configuration loading for publish rules that explicitly set
|
||||
requiredness while relying on a derived destination.
|
||||
|
||||
## Summary
|
||||
|
||||
Publish rules with a derivable source and omitted `dest` now load correctly
|
||||
whether `required` is omitted, explicitly true, or explicitly false. Previously,
|
||||
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
|
||||
diagnostic instead of accepting valid configuration.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a backward-compatible bookkeeping correction. Publish schemas,
|
||||
destination derivation, requiredness defaults, and YAML syntax support are
|
||||
unchanged. Explicit destinations and explicit false values remain intact.
|
||||
No ConfigKit upgrade or publication/storage behavior change is involved.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special upgrade action or configuration migration is required. Operators
|
||||
need not add a derivable `dest`, remove explicit `required` values, or change
|
||||
flow-style YAML to block-style YAML. See the
|
||||
[configuration reference](../config.md#publish-configuration-summary).
|
||||
|
||||
## Changes
|
||||
|
||||
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
|
||||
while retaining field-level resolution records.
|
||||
- Added real load/validation and safe effective-configuration regression coverage
|
||||
for built-in and configured-artifact sources, explicit requiredness, and an
|
||||
explicit destination that must remain unchanged.
|
||||
26
docs/releases/v2.1.2.md
Normal file
26
docs/releases/v2.1.2.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Narratio v2.1.2
|
||||
|
||||
This release allows secrets directories to contain hidden metadata without
|
||||
preventing credential loading.
|
||||
|
||||
## Summary
|
||||
|
||||
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
|
||||
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
|
||||
valid credentials from loading and are omitted from bootstrap reports.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a backward-compatible dependency bug fix. Configuration schemas,
|
||||
existing-environment precedence, and validation of non-hidden entries are
|
||||
unchanged. Hidden entries still count toward the directory entry limit.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special upgrade action or configuration migration is required. See the
|
||||
[credential configuration reference](../config.md) for secrets-directory usage.
|
||||
|
||||
## Changes
|
||||
|
||||
- Updated ConfigKit from v0.3.0 to v0.3.1.
|
||||
- Clarified hidden-entry handling in the secrets-directory documentation.
|
||||
53
docs/releases/v2.2.0.md
Normal file
53
docs/releases/v2.2.0.md
Normal file
@@ -0,0 +1,53 @@
|
||||
# Narratio v2.2.0
|
||||
|
||||
This release adds resumable per-speaker transcription, replayable WhisperX
|
||||
uploads, and explicit Notarius configuration profile selection.
|
||||
|
||||
## Summary
|
||||
|
||||
Successfully checkpointed speaker transcripts survive later transcription
|
||||
failures. Ordinary retry reuses valid completed speakers and processes only the
|
||||
remaining work. WhisperX uploads can now be replayed by the HTTP transport
|
||||
without buffering whole audio files.
|
||||
|
||||
## Compatibility
|
||||
|
||||
Existing CLI commands, speaker identities, transcript formats, and pipeline
|
||||
ordering are unchanged. Session manifests gain optional per-speaker completion
|
||||
evidence. Older manifests remain readable, but older Narratio binaries may
|
||||
reject manifests containing the new fields; downgrading after transcription
|
||||
is not transparent.
|
||||
|
||||
The optional `pipeline.notarius.config_profile` field selects a downstream
|
||||
Notarius configuration profile independently of Narratio's own profile. Using
|
||||
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
|
||||
supporting version. Omitting the field retains downstream default selection.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No configuration migration is required. When transcribe is next selected for
|
||||
an older aggregate-only manifest, all current speakers are transcribed once
|
||||
to establish reusable evidence. Previously unrecorded results from failed runs
|
||||
are not automatically imported.
|
||||
|
||||
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
|
||||
reuse of prior results; an ordinary retry after a failed forced invocation can
|
||||
retain replacements already checkpointed by that attempt. See the
|
||||
[operations guide](../operations.md) and
|
||||
[Notarius profile configuration](../config.md#notarius-configuration-profile).
|
||||
|
||||
## Changes
|
||||
|
||||
- Persist validated speaker completions independently of aggregate stage status,
|
||||
with serialized saves for concurrent work and input/output integrity checks.
|
||||
- Rebuild aggregate transcript references when speakers are removed or those
|
||||
references need repair, without retranscribing otherwise-current speakers.
|
||||
- Keep removed and incomplete speaker results out of versioned merge inputs.
|
||||
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
|
||||
service rejection still fails normally.
|
||||
- Provide fresh streaming multipart bodies for HTTP transport replay while
|
||||
preserving application retries, cancellation, and resource cleanup. This
|
||||
addresses the missing-body-replay limitation, not persistent upstream
|
||||
protocol failures or exactly-once service execution.
|
||||
- Pass explicit Notarius configuration profiles through the CLI and include
|
||||
their selection in extraction reuse decisions.
|
||||
35
docs/releases/v2.2.1.md
Normal file
35
docs/releases/v2.2.1.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Narratio v2.2.1
|
||||
|
||||
This maintenance release consolidates checkpoint persistence for transcription
|
||||
and analysis without changing their operator-facing behavior.
|
||||
|
||||
## Summary
|
||||
|
||||
Transcribe and analyze now share one internal operation for preparing isolated
|
||||
checkpoint candidates, saving them, and promoting successfully saved progress
|
||||
into live state. Each stage retains its own progress model and recovery policy.
|
||||
|
||||
## Compatibility
|
||||
|
||||
CLI commands, configuration, manifest schemas, artifact formats, and external
|
||||
integration contracts are unchanged. Speaker concurrency, artifact selection,
|
||||
force behavior, and ordinary resume semantics are preserved. No dependency
|
||||
upgrade or new external-tool version requirement is introduced.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special upgrade action or configuration migration is required when upgrading
|
||||
from v2.2.0. See the [v2.2.0 release notes](v2.2.0.md) when upgrading from an
|
||||
earlier version for the original speaker-checkpoint upgrade considerations.
|
||||
|
||||
## Changes
|
||||
|
||||
- Replace duplicate analyze/transcribe save-and-promote sequences with a
|
||||
shared, private checkpoint helper.
|
||||
- Isolate checkpoint collections from live state, including analyze records
|
||||
normalized during a transcription checkpoint save.
|
||||
- Preserve transcription's caller-owned locking and failure handling, and
|
||||
analysis's working/committed collections and rollback behavior.
|
||||
- Add focused checkpoint integrity tests while retaining stage and application
|
||||
recovery coverage.
|
||||
- Document the shared [checkpoint persistence boundary](../internal/manifest.md#checkpoint-persistence-boundary).
|
||||
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
@@ -0,0 +1,102 @@
|
||||
---
|
||||
title: Select a Notarius configuration profile through Narratio
|
||||
status: proposed
|
||||
consumer: rakestraw.dnd Ansible collection and its pipeline role
|
||||
|
||||
use_case: >-
|
||||
The Ansible pipeline role deploys Narratio and Notarius as independently
|
||||
configured applications. Operators select Narratio's testing profile to
|
||||
develop prompts and other pipeline assets with inexpensive models while
|
||||
retaining production settings in the base configuration. That selection
|
||||
currently cannot select a Notarius configuration profile, leaving extraction
|
||||
on its production settings.
|
||||
|
||||
desired_outcome: >-
|
||||
Narratio's testing overlay can explicitly select Notarius's testing overlay.
|
||||
Notarius owns the resulting model choices and any other testing-specific
|
||||
settings. The operator selects testing once when invoking Narratio, without
|
||||
editing production files or separately invoking Notarius.
|
||||
|
||||
rationale: >-
|
||||
Both applications already use ConfigKit configuration overlays. Selecting
|
||||
the downstream overlay preserves each application's configuration ownership
|
||||
and supports future Notarius testing adjustments without adding a Narratio
|
||||
configuration field for every downstream setting. This is explicit profile
|
||||
composition, not a requirement for arbitrary recursive orchestration.
|
||||
|
||||
requested_interface:
|
||||
field: pipeline.notarius.config_profile
|
||||
value: Optional Notarius ConfigKit configuration profile name
|
||||
supplied: >-
|
||||
Invoke Notarius with --config-profile and the selected name when extraction
|
||||
executes. The setting is available in Narratio's base configuration and
|
||||
its ordinary configuration overlays.
|
||||
omitted_or_null: >-
|
||||
Do not supply a profile-selection flag. Preserve existing Notarius
|
||||
configuration selection behavior.
|
||||
downstream_semantics: >-
|
||||
Retain Notarius's existing selector semantics, including the default
|
||||
selector for its base configuration. Notarius resolves profiles relative
|
||||
to its own configuration root, not Narratio's root.
|
||||
explicit_selection: >-
|
||||
Do not automatically forward Narratio's profile name. The selected
|
||||
Notarius profile may have the same name or a different name.
|
||||
distinction: >-
|
||||
This selects a ConfigKit configuration overlay through --config-profile,
|
||||
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
|
||||
can itself select the desired LLM profiles using native Notarius settings.
|
||||
|
||||
illustrative_consumer_configuration:
|
||||
note: >-
|
||||
These are separate files managed by the Ansible suite, not a proposed
|
||||
combined configuration document. The Notarius pipeline and LLM profile
|
||||
names are illustrative; the referenced LLM profile must be available to
|
||||
Notarius.
|
||||
narratio_profiles_testing_yml: |
|
||||
notarius:
|
||||
config_profile: testing
|
||||
notarius_profiles_testing_yml: |
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: gemini-flash-latest
|
||||
operator_workflow: >-
|
||||
Select --profile testing on the ordinary Narratio invocation. Narratio
|
||||
selects Notarius's testing configuration overlay when it runs extraction;
|
||||
other Narratio testing overrides continue to apply normally.
|
||||
|
||||
observable_behavior:
|
||||
- >-
|
||||
Notarius owns overlay discovery, composition, validation, and the effective
|
||||
downstream settings. A missing or invalid selected overlay must not silently
|
||||
fall back to production or be reported as successful extraction.
|
||||
- >-
|
||||
Changing the selected Notarius configuration profile must not silently
|
||||
reuse Narratio's completed extraction from the previous selection.
|
||||
Existing downstream invalidation behavior applies when extraction changes.
|
||||
- >-
|
||||
Editing Notarius overlay contents behind an unchanged selector retains
|
||||
the existing operator responsibility to force extraction when needed.
|
||||
This request does not require inspecting or tracking private Notarius files.
|
||||
- >-
|
||||
Existing credential forwarding remains independently configurable through
|
||||
notarius.llm_api_key, including in the Narratio testing overlay when the
|
||||
selected Notarius settings require a different credential reference.
|
||||
|
||||
consumer_follow_up: >-
|
||||
After Narratio releases this interface, the standalone Narratio Ansible role
|
||||
can expose an optional variable and omit the field when unset. The suite role
|
||||
can deploy Notarius's testing overlay and select it from Narratio's testing
|
||||
overlay, then distribute the changes through the collection. These consumer
|
||||
updates are separate from this upstream feature request.
|
||||
|
||||
scope_boundaries:
|
||||
- No Notarius change is currently required; v0.7.0 supports --config-profile.
|
||||
- No separate Narratio-side Notarius LLM-profile selector is requested.
|
||||
- No automatic profile-name propagation or downstream configuration rewriting.
|
||||
- No new credential discovery, copying, or provider-name remapping.
|
||||
- This request specifies consumer behavior, not internal Narratio code changes.
|
||||
|
||||
references:
|
||||
narratio_configuration: ../config.md
|
||||
narratio_notarius_integration: ../integrations/notarius.md
|
||||
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md
|
||||
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
@@ -0,0 +1,171 @@
|
||||
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
|
||||
|
||||
Status: Implemented; release target v2.1.1
|
||||
|
||||
## Summary
|
||||
|
||||
Narratio should successfully derive a publish destination when a valid
|
||||
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
|
||||
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
|
||||
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
|
||||
|
||||
This is a configuration-resolution bookkeeping defect, not a YAML formatting
|
||||
restriction. The fix should preserve the existing public configuration
|
||||
contract: operators may omit a derivable destination and may independently set
|
||||
`required` to either `true` or `false`.
|
||||
|
||||
## Observed Behavior
|
||||
|
||||
A pipeline containing a rule equivalent to the following fails during native
|
||||
configuration loading:
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
outputs: [{"source": "narratio.transcript.final", "required": true}]
|
||||
```
|
||||
|
||||
Narratio reports:
|
||||
|
||||
```text
|
||||
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
|
||||
```
|
||||
|
||||
The same failure is possible with ordinary block-style YAML when `required` is
|
||||
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
|
||||
cause of the defect.
|
||||
|
||||
The existing destination-derivation tests do not expose this case because
|
||||
their rules also omit `required`. Narratio then supplies the default value for
|
||||
`required` and records a change covering the complete `publish.outputs`
|
||||
sequence. That broader record happens to cover the destination mutation as
|
||||
well, masking the missing resolution record that occurs when `required` is
|
||||
already explicit.
|
||||
|
||||
## Expected Contract
|
||||
|
||||
The current configuration contract remains unchanged:
|
||||
|
||||
- `publish.outputs[].source` is required;
|
||||
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
|
||||
selected built-in or configured artifact source;
|
||||
- `publish.outputs[].required` defaults to `true`, but an operator may set it
|
||||
explicitly to either `true` or `false`; and
|
||||
- equivalent valid YAML representations have equivalent configuration
|
||||
semantics.
|
||||
|
||||
For example, both rules below should load and validate, retain their explicit
|
||||
requiredness, and receive the canonical derived destination:
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
outputs:
|
||||
- source: narratio.transcript.final
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
required: false
|
||||
```
|
||||
|
||||
## Root Cause
|
||||
|
||||
ConfigKit intentionally treats sequences as atomic when comparing values
|
||||
before and after application resolution. Changing a field within one
|
||||
`publish.outputs` entry is therefore observed as a change to the
|
||||
`/publish/outputs` sequence.
|
||||
|
||||
Narratio currently derives each missing destination by recording a
|
||||
normalization at the individual `publish.outputs[index].dest` path. That child
|
||||
record does not cover the atomic parent-sequence change. A separate
|
||||
container-level record is currently created only when Narratio also defaults
|
||||
an omitted `required` value, so configurations with explicit `required` values
|
||||
lose that incidental coverage.
|
||||
|
||||
## Recommended Fix
|
||||
|
||||
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
|
||||
atomic-sequence contract. Whenever resolution derives one or more destinations
|
||||
inside `publish.outputs`, the recorded resolution changes must cover the
|
||||
`/publish/outputs` container. Destination derivation must not depend on an
|
||||
unrelated `required` default being applied in the same list.
|
||||
|
||||
The implementation may retain more specific provenance where useful, but it
|
||||
must provide valid container-level coverage for the sequence mutation. Do not
|
||||
change ConfigKit's sequence comparison semantics merely to accommodate this
|
||||
call site.
|
||||
|
||||
## Scope
|
||||
|
||||
- Correct destination derivation for publish rules that explicitly set
|
||||
`required` and omit a derivable `dest`.
|
||||
- Preserve explicit `required: true` and `required: false` values.
|
||||
- Preserve canonical destination derivation for built-in transcript sources
|
||||
and configured artifact sources.
|
||||
- Add focused regression coverage through Narratio's real configuration load
|
||||
and validation boundary.
|
||||
- Prepare a backward-compatible patch release after the fix passes the normal
|
||||
repository validation.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
|
||||
destination derivation rules.
|
||||
- Requiring operators or configuration generators to supply `dest` when it is
|
||||
derivable.
|
||||
- Requiring operators to omit explicit `required: true` values.
|
||||
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
|
||||
- Changing ConfigKit's documented atomic treatment of sequences.
|
||||
- Adding storage, publication, or network integration behavior.
|
||||
|
||||
## Regression Coverage
|
||||
|
||||
Add one focused package-level behavioral test that loads and validates a
|
||||
pipeline with omitted destinations and explicit requiredness. A representative
|
||||
list should include both `required: true` and `required: false`, then assert
|
||||
that:
|
||||
|
||||
- loading and validation succeed;
|
||||
- each destination is derived from its source; and
|
||||
- each explicit `required` value is preserved.
|
||||
|
||||
Using the compact flow-style representation that exposed the integration
|
||||
failure is reasonable, while the existing block-style cases continue to cover
|
||||
omitted `required`. The test should exercise the public configuration behavior,
|
||||
not ConfigKit call choreography or Narratio's private recording mechanism.
|
||||
|
||||
No network service, object store, credential, or test double is required.
|
||||
|
||||
## Target End State
|
||||
|
||||
All valid publish rules with derivable sources behave consistently across
|
||||
these cases:
|
||||
|
||||
| `dest` | `required` | Result |
|
||||
| --- | --- | --- |
|
||||
| omitted | omitted | destination derived; required defaults to `true` |
|
||||
| omitted | `true` | destination derived; explicit `true` retained |
|
||||
| omitted | `false` | destination derived; explicit `false` retained |
|
||||
| explicit | either value | explicit destination and requiredness retained |
|
||||
|
||||
`narratio config validate`, `narratio config show`, and ordinary pipeline
|
||||
loading accept the corrected cases without an unrecorded-change diagnostic.
|
||||
No operator configuration migration or workaround is required.
|
||||
|
||||
## Release
|
||||
|
||||
This is a backward-compatible correction to documented configuration behavior
|
||||
and should be released as Narratio `v2.1.1`. The release note should identify
|
||||
the corrected explicit-required/derived-destination interaction and state that
|
||||
no special upgrade action is required.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- The focused regression test fails against v2.1.0 behavior and passes after
|
||||
the fix.
|
||||
- Publish rules with explicit `required: true` or `required: false` and omitted
|
||||
derivable destinations load and validate successfully.
|
||||
- Derived destinations and explicit requiredness appear correctly in the safe
|
||||
effective configuration.
|
||||
- Existing destination derivation, publish validation, imports, profiles, and
|
||||
configuration inspection tests remain green.
|
||||
- The complete repository validation and release-candidate checks pass.
|
||||
- The correction is published as the immutable `v2.1.1` tag with the required
|
||||
release note.
|
||||
333
docs/roadmap/implementation.md
Normal file
333
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,333 @@
|
||||
# Implementation Plan: Shared Stage Checkpoint Mechanics
|
||||
|
||||
Status: Complete.
|
||||
|
||||
## Scope and Authority
|
||||
|
||||
Implement the agreed follow-up to v2.2.0: consolidate the thin checkpoint
|
||||
mechanism shared by transcribe and analyze without combining their progress
|
||||
models or changing operator behavior. There is no separate consolidation
|
||||
roadmap in this checkout. This plan uses the accompanying discussion as its
|
||||
scope: persistence is already centralized; candidate isolation and
|
||||
save-before-promotion are the duplication to remove. The completed
|
||||
[transcription roadmap](transcribe-checkpoints-and-replay.md) provides background,
|
||||
not authorization to reimplement transcription or HTTP replay. The preceding
|
||||
completed implementation plan remains in Git history.
|
||||
|
||||
Complete the numbered stages in order, one bounded implementation prompt per
|
||||
stage for gpt-5.6-terra. Record completion and actual verification results here.
|
||||
This planning pass changes documentation only; the steps below are for the
|
||||
subsequent implementation sprint.
|
||||
|
||||
Before each stage, read `AGENTS.md`, `docs/development.md`, and all three
|
||||
documents in `docs/policy/`: architecture, documentation, and testing. Use
|
||||
maintained Go 1.26. Prefer graph tools for code discovery; inspect current
|
||||
files if their index is stale. Read the stage-specific preparation below,
|
||||
rather than unrelated historical roadmaps.
|
||||
|
||||
## Target and Scope Boundaries
|
||||
|
||||
The existing `internal/manifest.Store` and `LocalStore` remain the session
|
||||
manifest persistence interface and implementation. Encoding, schemas,
|
||||
validation, normalization, and atomic file replacement remain unchanged.
|
||||
There is no new store, journal, file format, dependency, command, configuration,
|
||||
or migration.
|
||||
|
||||
One private helper in `internal/stage` owns candidate preparation, calling the
|
||||
existing store, and promoting the selected stage record only after successful
|
||||
save. The existing typed analyze/transcribe checkpoint methods become thin
|
||||
callers. This is execution plumbing: keep it beside the callers rather than
|
||||
moving orchestration into the storage package or exporting a reusable API.
|
||||
|
||||
Keep these responsibilities stage-owned:
|
||||
|
||||
- Transcribe's completed-only speaker map, evidence checks, bounded concurrency,
|
||||
mutex, checkpoint-failure latch, cancellation, pre-dispatch force reset,
|
||||
output installation, and aggregate repair.
|
||||
- Analyze's dispositions, selection, dependencies, catalog updates, working and
|
||||
committed collections, failure reconciliation, invalidation, and outputs.
|
||||
- Each stage's decision whether/when to checkpoint, typed validation, contextual
|
||||
errors, and post-save working-state updates.
|
||||
- Runner aggregate transitions, session locking, terminal failure saves, and
|
||||
fixed dependent-stage invalidation.
|
||||
|
||||
Do not introduce a common subtask model, scheduler, transaction manager, global
|
||||
mutex, automatic retry, additional checkpoint, or batching policy. Do not
|
||||
migrate publish/restore, planning clones, cleanup, or ordinary runner saves:
|
||||
their operations are not the demonstrated duplication. Leave HTTP replay and
|
||||
all adapters unchanged.
|
||||
|
||||
## Selected Design
|
||||
|
||||
### Shared Save-and-Promote Operation
|
||||
|
||||
Add `internal/stage/checkpoint.go` with a private helper shaped as:
|
||||
|
||||
```go
|
||||
func checkpointStageCollection(
|
||||
ctx context.Context,
|
||||
store manifest.Store,
|
||||
path string,
|
||||
live *manifest.Manifest,
|
||||
stageName string,
|
||||
update func(*manifest.StageRecord) error,
|
||||
) error
|
||||
```
|
||||
|
||||
Names are implementation choices, not public contracts. Use a function and
|
||||
small copying helpers, not a factory/options/interface framework. The two
|
||||
existing typed checkpoint methods are the only production callers. They pass
|
||||
the already-resolved manifest path; the helper does not discover paths or
|
||||
inspect configuration.
|
||||
|
||||
The helper performs this sequence:
|
||||
|
||||
1. Check required live/store inputs without mutation. Copy a checkpoint
|
||||
candidate and select a copied target record. If absent, provide an empty
|
||||
candidate record to the caller, without inferring its name, status, or time.
|
||||
2. Apply the supplied update to that candidate record only. On update error,
|
||||
return without saving or promoting. The typed caller validates and fills
|
||||
its version, collection, and stage-owned initialization/timestamps.
|
||||
3. Invoke `Store.Save` once with the supplied context. Preserve error identity
|
||||
through `%w` and useful path context. On error, do not promote, retry,
|
||||
reload, or synthesize aggregate failure.
|
||||
4. After success, install the saved target record and candidate manifest
|
||||
`UpdatedAt` into live state. Preserve an existing target record's pointer
|
||||
identity by assigning through it. Allocate missing live maps/records only
|
||||
after success. Do not replace the entire live manifest or overwrite other
|
||||
stage records. Copy mutable checkpoint collections on promotion so saved
|
||||
candidates and caller working collections cannot mutate live progress.
|
||||
|
||||
Callers serialize the whole operation. Transcribe's current mutex does so;
|
||||
analyze executes artifacts sequentially. Do not add a lock to the helper or
|
||||
promise support for concurrent independent manifest writers. Atomic replacement
|
||||
visibility does not become power-loss durability or a transaction covering
|
||||
transcript files.
|
||||
|
||||
### Bounded Candidate Isolation
|
||||
|
||||
Factor `cloneManifestForTranscribeCheckpoint` into a shared checkpoint-scoped
|
||||
copy helper in the new file. Use ordinary typed copies and existing
|
||||
`manifest.CloneAnalyzeArtifactCollection` and
|
||||
`manifest.CloneTranscribeSpeakerCollection`. Do not introduce reflection
|
||||
cloning, JSON round trips, or a public general-purpose clone API.
|
||||
|
||||
Copy the manifest value, stages map, stage records, and mutable collections
|
||||
touched by these updates or `LocalStore.Save`. In particular, Save normalizes
|
||||
analyze collections throughout the candidate, including on a transcribe
|
||||
checkpoint. Clone those collections for every stage so normalization cannot
|
||||
leak into live state on failure. Preserve the existing bounded copies of
|
||||
input/artifact and output/log/config slices. Retain nil entries for existing
|
||||
validation to reject, rather than silently repairing invalid manifests.
|
||||
|
||||
This is not arbitrary deep isolation of metadata. Neither the two update
|
||||
callbacks nor the current Store mutates `map[string]any` metadata, so it may
|
||||
remain shared read-only. Document that boundary and inspect every current
|
||||
normalization write. Use existing typed clones for nested checkpoint evidence;
|
||||
do not restrict legal metadata values or alter their Go/JSON representations
|
||||
to simplify copying.
|
||||
|
||||
### Differences to Preserve
|
||||
|
||||
- Analyze still requires an existing stage record. Its callback changes only
|
||||
analyze version/collection, not transcribe-style initialization or timestamp
|
||||
policy. It refreshes working and committed collections only after success.
|
||||
- Transcribe retains missing-record initialization for direct stage calls:
|
||||
its typed caller supplies name, running status, and initial timestamps when
|
||||
needed. Its callback retains the existing stage `UpdatedAt` update. Copying
|
||||
an existing record must not clear aggregate fields or invoke lifecycle
|
||||
transitions.
|
||||
- Existing transcribe live-record initialization is incomplete on the unusual
|
||||
missing-record path; promoting the actual successfully saved candidate makes
|
||||
its identity agree with disk. This does not add a new missing-record
|
||||
rejection or change ordinary runner status transitions.
|
||||
- Analyze rollback/failure handling and transcribe failure latching remain in
|
||||
their execution owners. The shared operation never invents failed items.
|
||||
- Preserve starting-checkpoint elision, force/reset semantics, empty versioned
|
||||
collections, evidence and producer identity, legacy transcription handling,
|
||||
supported zero-sized audio, and aggregate repair without HTTP calls.
|
||||
- Do not freeze exact error wording, wall-clock values, copy choreography, or
|
||||
iteration order into new contracts. No new schema, limits, filename rules,
|
||||
statuses, or compatibility guarantees are authorized.
|
||||
|
||||
## Stage 1: Add the Shared Checkpoint Operation
|
||||
|
||||
Status: Complete.
|
||||
|
||||
Completion marker (2026-09-22): Added the private stage-level checkpoint
|
||||
save-and-promote helper and bounded candidate copying without migrating either
|
||||
production caller. Focused real-store coverage proves live isolation through
|
||||
updates, validation, save-time normalization and timestamps, save failures,
|
||||
missing-target creation, pointer-preserving promotion, unrelated aggregate
|
||||
preservation, caller-copy isolation, context/error identity, and exact one-save
|
||||
behavior. Verified with `go test ./internal/manifest ./internal/stage`,
|
||||
`go test -race ./internal/stage`, and `git diff --check`.
|
||||
|
||||
Preparation: `internal/manifest/store.go`, `manifest.go`, `analyze_state.go`,
|
||||
`transcribe_state.go`; `internal/stage/analyze_execution_state.go`,
|
||||
`transcribe_execution_state.go`; `docs/internal/manifest.md`.
|
||||
|
||||
1. Add the private helper and bounded candidate copying described above.
|
||||
Leave existing production checkpoint paths unchanged until their migration
|
||||
stages. Do not duplicate manifest record-validation rules.
|
||||
2. Add focused `internal/stage/checkpoint_test.go` coverage with real
|
||||
`manifest.LocalStore` and temporary directories for successful save/reload.
|
||||
Use a small Store wrapper only to observe/fail the persistence boundary.
|
||||
3. Prove live progress remains old during Save, matches saved progress after
|
||||
success, and does not change on update, validation, or save failure. Include
|
||||
both typed collections and a transcribe checkpoint alongside analyze
|
||||
records subject to save-time normalization. A failure test must exercise
|
||||
candidate timestamp/normalization mutation before returning an error, not
|
||||
merely fail before any work.
|
||||
4. Prove unrelated stage/aggregate data survives, existing target references
|
||||
observe updates, missing-target creation is invisible on failure, and later
|
||||
caller collection edits do not mutate committed records. Preserve context
|
||||
and error identity. Test the supported copy/write boundary, not arbitrary
|
||||
malicious Store mutation.
|
||||
|
||||
Verify: `go test ./internal/manifest ./internal/stage` and
|
||||
`go test -race ./internal/stage`.
|
||||
|
||||
Done when the shared integrity boundary is tested without changing existing
|
||||
stage behavior. Temporary copying overlap is removed in Stage 3.
|
||||
|
||||
## Stage 2: Migrate Analyze
|
||||
|
||||
Status: Complete.
|
||||
|
||||
Completion marker (2026-09-22): Replaced analyze's local candidate-copy,
|
||||
save, and promotion choreography with the shared checkpoint operation while
|
||||
retaining its existing-record requirement, typed validation, path resolution,
|
||||
and post-save working and committed collection refresh. Existing recovery,
|
||||
selection, invalidation, and rollback coverage required no duplication.
|
||||
Verified with `go test ./internal/stage ./internal/app ./internal/manifest`,
|
||||
`go test -race ./internal/stage ./internal/app`, and `git diff --check`.
|
||||
|
||||
Preparation: `internal/stage/analyze_execution_state.go`,
|
||||
`analyze_checkpoint_test.go`, `analyze_failure_test.go`,
|
||||
`internal/app/analyze_checkpoint_recovery_test.go`, and
|
||||
`docs/internal/stage-analyze.md`.
|
||||
|
||||
1. Replace manual candidate copying, saving, and live promotion in analyze
|
||||
with the shared helper. Retain typed validation, required-record checking,
|
||||
the typed update callback, and committed/working collection refresh.
|
||||
2. Preserve starting-checkpoint elision, selected-artifact behavior, catalog
|
||||
sequencing, dependent invalidation on changed output, and rollback after
|
||||
checkpoint failure. Do not change the runner's terminal failure save or
|
||||
joined errors.
|
||||
3. Run existing tests before adding coverage. Retain tests proving earlier
|
||||
artifacts survive failure/cancellation, unsaved output bytes gain no
|
||||
authority, selected/forced work preserves appropriate unrelated artifacts,
|
||||
and interrupted runs recover across store reloads. Add only missing
|
||||
integration coverage; do not duplicate Stage 1's isolation matrix.
|
||||
|
||||
Verify: `go test ./internal/stage ./internal/app ./internal/manifest` and
|
||||
`go test -race ./internal/stage ./internal/app`.
|
||||
|
||||
Done when analyze delegates persistence choreography while progress, artifact
|
||||
selection, and failure behavior remain unchanged.
|
||||
|
||||
## Stage 3: Migrate Transcribe and Remove Duplicate Machinery
|
||||
|
||||
Status: Complete.
|
||||
|
||||
Completion marker (2026-09-22): Delegated transcription's locked checkpoint
|
||||
operation to the shared helper while retaining typed validation, direct-call
|
||||
record initialization, stage timestamp updates, post-save working collection
|
||||
refresh, and the existing caller-owned mutex and failure latch. Removed the
|
||||
duplicate production manifest clone; the read-only resume assertion now uses
|
||||
the shared bounded snapshot. Existing real-store, concurrency, force/retry,
|
||||
interruption, aggregate-repair, and zero-sized-audio coverage passed unchanged.
|
||||
Verified with `go test ./internal/stage ./internal/app ./internal/manifest`,
|
||||
`go test -race ./internal/stage ./internal/app`, and `git diff --check`.
|
||||
|
||||
Preparation: `internal/stage/transcribe_execution_state.go`, `transcribe.go`,
|
||||
`transcribe_resume.go`, `transcribe_checkpoint_test.go`,
|
||||
`transcribe_reconciliation_test.go`,
|
||||
`internal/app/transcribe_checkpoint_recovery_test.go`, and
|
||||
`docs/internal/stage-transcribe.md`.
|
||||
|
||||
1. Delegate `checkpointCollectionLocked` to the shared helper, retaining
|
||||
typed initialization/update and successful working-collection refresh.
|
||||
Keep the existing mutex across candidate creation, save, and promotion.
|
||||
Do not move HTTP/hashing work into that critical section or add a lock.
|
||||
2. Preserve failure-latch checks and worker joining, the parent versus worker
|
||||
context distinction, force's durable empty starting collection, and
|
||||
ordinary retry's reuse of committed forced replacements.
|
||||
3. Remove `cloneManifestForTranscribeCheckpoint`. Its current use by a
|
||||
read-only resume test is not a reason to retain duplicate production code.
|
||||
Use an independent test snapshot or the shared bounded copy where sufficient,
|
||||
without exporting a helper for tests.
|
||||
4. Retain existing real-store and concurrency coverage: no lost completions;
|
||||
pre-dispatch save failure prevents requests/canonical writes; later failure
|
||||
retains prior authority; interrupted and aggregate-save recovery; failed
|
||||
force followed by retry; run-directory independence; zero-sized audio;
|
||||
removed/mismatched aggregate repair without additional WhisperX requests
|
||||
or removed-speaker merge input. Do not weaken these regression assertions.
|
||||
|
||||
Verify: `go test ./internal/stage ./internal/app ./internal/manifest` and
|
||||
`go test -race ./internal/stage ./internal/app`.
|
||||
|
||||
Done when both stages use one save/promote implementation and one bounded
|
||||
copy implementation, with their different models and synchronization intact.
|
||||
|
||||
## Stage 4: Document, Review Scope, and Validate
|
||||
|
||||
Status: Complete.
|
||||
|
||||
Completion marker (2026-09-22): Documented the private shared checkpoint
|
||||
persistence boundary and linked the stage guides back to it while retaining
|
||||
their separate analyze and transcribe progress policies. The refactor removed
|
||||
the two local candidate/save/promotion sequences and the transcription-only
|
||||
bounded clone; the retained thin wrappers own typed validation, updates,
|
||||
working-state refresh, and their existing serialization and failure behavior.
|
||||
The collective test review retained focused helper integrity coverage plus
|
||||
stage/application recovery tests, without duplicating the helper's isolation
|
||||
matrix. A cumulative scope audit found no schema, validation-policy,
|
||||
scheduling, force, concurrency, output-selection, stage-order, adapter-call,
|
||||
configuration, or public-interface change.
|
||||
|
||||
Verified with `go test ./...`, `go test -race ./...`, `go vet ./...`,
|
||||
`go build ./...`, `go test ./internal/doccheck`,
|
||||
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||
`git diff --check`; all passed.
|
||||
|
||||
Preparation: `docs/internal/manifest.md`, `docs/internal/stage-transcribe.md`,
|
||||
`docs/internal/stage-analyze.md`, documentation/testing policies, and the
|
||||
implementation diff against this plan's starting commit.
|
||||
|
||||
1. Update manifest internals to identify the shared checkpoint owner and copy/
|
||||
save/promote boundary. Stage guides retain stage-specific policy and link
|
||||
to that explanation instead of duplicating shared implementation details.
|
||||
Do not rewrite immutable release notes or announce new operator behavior.
|
||||
2. Assess tests collectively. Consolidate genuinely duplicated persistence
|
||||
mechanics coverage, but retain stage/application recovery tests proving
|
||||
correct integration. Avoid test-count/LOC quotas. Record a brief summary
|
||||
here of duplicated mechanisms removed and thin wrappers retained.
|
||||
3. Confirm no schema, validation policy, scheduling, force, concurrency, output
|
||||
selection, stage order, adapter call, or configuration change slipped into
|
||||
the refactor. Additional restrictions require a separate decision, not
|
||||
tests that retrospectively declare new policy.
|
||||
4. Run all gates below and record actual results. Mark this plan complete only
|
||||
when both migrations and verification pass. Do not release, tag, push, delete
|
||||
other roadmaps, or expand into a repository-wide cloning refactor.
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go test ./internal/doccheck
|
||||
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Done when duplication is removed, recovery remains covered, documentation
|
||||
matches the implementation, and all gates pass.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None requiring a product decision. A private stage-level helper is selected
|
||||
because persistence already has an owner and the duplication is execution
|
||||
plumbing. Model unification, library extraction, generalized transactions, and
|
||||
broader cloning cleanup remain out of scope.
|
||||
File diff suppressed because it is too large
Load Diff
151
docs/roadmap/notarius-config-profile.md
Normal file
151
docs/roadmap/notarius-config-profile.md
Normal file
@@ -0,0 +1,151 @@
|
||||
# Notarius Configuration Profile Selection
|
||||
|
||||
Status: Implemented.
|
||||
|
||||
## Goal and Authority
|
||||
|
||||
Implement the consumer behavior in the
|
||||
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
|
||||
configuration overlay explicitly select a Notarius configuration overlay.
|
||||
Operators can select testing once in Narratio while each application retains
|
||||
ownership of its own model choices and other settings.
|
||||
|
||||
This roadmap describes the target state and required change areas, not an
|
||||
implementation sequence; the [implementation plan](implementation.md) owns
|
||||
the ordered work and implementation decisions. The feature request is the scope
|
||||
authority; the mechanisms suggested below do not establish additional
|
||||
user-facing contracts.
|
||||
|
||||
## Underlying Issue
|
||||
|
||||
Narratio can select its own testing profile, but extraction currently invokes
|
||||
Notarius without a configuration-profile selector. Consequently, a Narratio
|
||||
testing invocation cannot explicitly select the independently deployed Notarius
|
||||
testing overlay through pipeline configuration.
|
||||
|
||||
Adding individual downstream model or tuning fields would duplicate Notarius's
|
||||
configuration responsibilities. Passing its existing profile selector provides
|
||||
the requested flexibility without that duplication.
|
||||
|
||||
## Targeted End State
|
||||
|
||||
### Explicit Configuration Composition
|
||||
|
||||
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
|
||||
name, usable in Narratio's base configuration and ordinary overlays. For
|
||||
example, Narratio's `profiles/testing.yml` may contain:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
config_profile: testing
|
||||
```
|
||||
|
||||
Selecting Narratio's `--profile testing` then selects that explicit downstream
|
||||
profile whenever extraction executes. The two profile names need not match.
|
||||
Narratio does not automatically forward its own profile name.
|
||||
|
||||
When supplied, the selection is passed through Notarius's `--config-profile`
|
||||
argument. Omitted or null values add no profile-selection flag and preserve
|
||||
existing downstream selection behavior. Notarius's selector semantics remain
|
||||
authoritative, including empty and `default` selections for the composed base
|
||||
configuration. Narratio does not introduce a stricter profile-name policy.
|
||||
|
||||
This field selects a ConfigKit configuration overlay, not a PromptKit profile
|
||||
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
|
||||
selection. Notarius resolves overlays relative to its own configuration root;
|
||||
Narratio neither resolves those files against its own root nor rewrites them.
|
||||
|
||||
### Execution, Reuse, and Failure
|
||||
|
||||
A changed downstream profile selection must not silently reuse Narratio's
|
||||
completed extraction from the previous selection. Existing dependent-stage
|
||||
invalidation applies when extraction changes; the feature does not change the
|
||||
pipeline order or transcript-generation behavior.
|
||||
|
||||
Notarius continues to own downstream overlay discovery, composition, validation,
|
||||
and cache compatibility. A missing or invalid selected overlay must fail
|
||||
extraction through the existing subprocess error path, without silently falling
|
||||
back to production or reporting successful extraction.
|
||||
|
||||
Existing resume and chunk-cache controls, including force precedence, remain
|
||||
in place. Profile selection does not itself mean forced regeneration or cache
|
||||
deletion. Narratio's decision to execute extraction and Notarius's decision to
|
||||
reuse compatible work remain separate responsibilities.
|
||||
|
||||
Changes to private Notarius configuration behind an unchanged selector retain
|
||||
the existing operator responsibility to force extraction when needed. There is
|
||||
no new downstream file inspection or content-tracking requirement.
|
||||
|
||||
Credential forwarding remains independently configured through
|
||||
`notarius.llm_api_key`. A Narratio testing overlay may select a different
|
||||
credential reference alongside the downstream profile; no new credential
|
||||
discovery or provider-name mapping is introduced.
|
||||
|
||||
## Required Change Areas
|
||||
|
||||
### Configuration and Inspection
|
||||
|
||||
Extend the typed Notarius configuration and its existing safe effective-YAML
|
||||
projection to carry the optional field. Preserve the ordinary configuration
|
||||
composition behavior for base values, overlay overrides, and null values.
|
||||
Configuration inspection must reflect the selected field without exposing
|
||||
credential values. No replacement configuration loader or downstream preflight
|
||||
loader is needed.
|
||||
|
||||
### Stage and Adapter Boundary
|
||||
|
||||
Carry the selection from resolved configuration through the extraction request
|
||||
to the subprocess adapter. Construct the optional CLI flag using the existing
|
||||
argument-vector mechanism, alongside the current configuration path, references,
|
||||
credential forwarding, and cache controls. Retain normal Notarius error handling.
|
||||
|
||||
### Extraction Reuse Evidence
|
||||
|
||||
Account for the downstream selection in the existing extraction reuse evidence.
|
||||
The current semantic-configuration and invocation fingerprints are the relevant
|
||||
change points. Keep their treatment of selection consistent so that a profile
|
||||
change cannot bypass the required rerun and downstream invalidation.
|
||||
|
||||
This roadmap does not require a new manifest schema, separate profile-indexed
|
||||
workspaces, historical per-profile result retention, or an upgrade-wide forced
|
||||
rerun.
|
||||
|
||||
### Verification and Documentation
|
||||
|
||||
Use focused behavioral coverage for configuration composition and inspection,
|
||||
explicit argument forwarding and omission, changed-selection reuse decisions,
|
||||
and ordinary subprocess failure handling. Reuse existing test coverage where
|
||||
it already protects unchanged credentials, force behavior, and invalidation;
|
||||
do not duplicate Notarius's complete profile-validation or cache test suites.
|
||||
|
||||
When implemented, update the canonical
|
||||
[configuration reference](../config.md),
|
||||
[Notarius integration contract](../integrations/notarius.md),
|
||||
[extract internals](../internal/stage-extract.md), and relevant maintained
|
||||
examples. The development workflow in [Operations](../operations.md) should
|
||||
explain explicit profile composition and the existing need to force extraction
|
||||
after private downstream edits. Clarify that profile selection does not create
|
||||
an isolated testing workspace or separate current extraction state.
|
||||
|
||||
## Compatibility and Scope Boundaries
|
||||
|
||||
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
|
||||
required. Document the capability needed when the option is used, without adding
|
||||
an unconditional runtime version gate for existing unconfigured deployments.
|
||||
|
||||
This feature adds no Narratio CLI selector, automatic profile propagation,
|
||||
Notarius LLM-profile selector, downstream configuration rewriting, new
|
||||
credential mechanism, or generalized recursive orchestration. Existing output
|
||||
and receipt contracts remain unchanged.
|
||||
|
||||
The consumer's Ansible role and suite updates are follow-up work outside this
|
||||
repository. They can deploy downstream overlays and populate this field after
|
||||
the interface is released.
|
||||
|
||||
## Scope Review
|
||||
|
||||
No stronger requirements or new invariants are proposed beyond the feature
|
||||
request and existing repository contracts. In particular, minimizing one-time
|
||||
upgrade invalidation and avoiding reruns between equivalent base selectors are
|
||||
implementation considerations, not additional acceptance guarantees imposed
|
||||
by this roadmap. No user decisions are currently required before planning.
|
||||
104
docs/roadmap/remediation.md
Normal file
104
docs/roadmap/remediation.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# Transcription Checkpoint Review Remediation
|
||||
|
||||
Status: Complete.
|
||||
|
||||
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
|
||||
upload-replay roadmap and the original user request. No production changes were
|
||||
made during review. The two failures below were reproduced with temporary tests
|
||||
using the existing stage fixtures; those temporary tests were removed afterward.
|
||||
|
||||
## 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
|
||||
|
||||
Priority: P2. Status: Complete.
|
||||
|
||||
Implemented read-only aggregate identity/evidence checks before resume skips.
|
||||
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
|
||||
repair without HTTP requests, downstream invalidation, and exclusion of removed
|
||||
speakers from merge without deleting their files. Equivalent path forms and
|
||||
output ordering remain reusable. Focused app/stage tests and documentation
|
||||
checks pass.
|
||||
|
||||
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
|
||||
consumed there.
|
||||
|
||||
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
|
||||
only establishes that the remaining speakers need no transcription; it does
|
||||
not establish that the stored speaker collection and aggregate outputs match
|
||||
the current input set. Removing a speaker while the remaining inputs are
|
||||
unchanged produces an empty work list, so a succeeded record can be skipped
|
||||
without pruning the removed speaker. Its old aggregate output can then remain
|
||||
available to merge. Likewise, missing/inconsistent aggregate output references
|
||||
are not repaired when all individual checkpoints remain valid. A prepare rerun
|
||||
may already stale transcribe in some workflows, but the resume validator cannot
|
||||
rely on that to satisfy its own current-input/output checks.
|
||||
|
||||
Reproduction: create complete alice/bob checkpoint records and aggregate
|
||||
outputs with the existing fixture, mark the stage succeeded, remove bob from
|
||||
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
|
||||
despite still retaining bob in the stored collection and aggregate outputs.
|
||||
|
||||
Required change:
|
||||
|
||||
- Distinguish no HTTP work from no reconciliation work. Validate that the
|
||||
committed speaker set and aggregate transcript references represent the
|
||||
reconciled current set, including relevant identity/evidence, before skipping.
|
||||
- When only collection/output repair is needed, return non-resumable and let
|
||||
normal execution checkpoint the pruned collection and rebuild aggregate
|
||||
outputs without retranscribing reusable speakers. Preserve existing dependent
|
||||
invalidation; do not force otherwise-current speaker jobs.
|
||||
- Keep resume inspection read-only. Do not fix this by deleting incidental
|
||||
transcript files or changing stage selection rules.
|
||||
|
||||
Verification: add removed-only input coverage (not just removal combined with
|
||||
an added speaker), plus incomplete/inconsistent aggregate output coverage.
|
||||
Exercise the runner skip path and assert zero additional WhisperX requests,
|
||||
correct rebuilt outputs, and no removed speaker reaching merge.
|
||||
|
||||
## 2. Remove the Unrequested Positive Audio-Size Requirement
|
||||
|
||||
Priority: P2. Status: Complete.
|
||||
|
||||
Audio evidence now permits zero size while still rejecting negative values.
|
||||
Regression tests verify zero-size manifest round trips, successful service
|
||||
results being checkpointed and reused, and service rejection being propagated
|
||||
without creating a completion record. Nonempty valid-JSON output validation is
|
||||
unchanged. Both new success regressions failed before the fix.
|
||||
|
||||
Location: `internal/manifest/transcribe_state.go:95`.
|
||||
|
||||
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
|
||||
a regular `.flac` file and delegated transcription acceptance to WhisperX;
|
||||
neither the user request nor the roadmap introduced minimum audio size or a new
|
||||
audio-content validation policy. Zero-byte content still has a valid checksum
|
||||
and can be represented by the checkpoint mechanism. The current check runs
|
||||
after a successful response and output materialization, causing the entire
|
||||
stage to fail instead of retaining the completed work.
|
||||
|
||||
Reproduction: prepare one speaker, replace its prepared audio with an empty
|
||||
regular file, and use the existing successful fake WhisperX client. Stage
|
||||
execution fails at checkpoint validation with `audio_size must be positive`.
|
||||
This demonstrates the extra Narratio restriction; it does not assert that a
|
||||
particular production WhisperX deployment accepts empty audio.
|
||||
|
||||
Required change:
|
||||
|
||||
- Permit zero audio size and reject negative sizes. Keep byte hashing and
|
||||
size equality checks. Let the downstream service decide whether the input
|
||||
can be transcribed, as before.
|
||||
- Do not remove the nonempty valid-JSON output requirement, which has a separate
|
||||
existing justification. Do not add a new early empty-audio rejection instead.
|
||||
- Update any new tests/documentation that promote strictly positive input size
|
||||
to a contract.
|
||||
|
||||
Verification: cover a zero-size record round-trip and successful adapter result
|
||||
being checkpointed/reused for that input; retain negative-size rejection and
|
||||
ordinary downstream-failure tests.
|
||||
|
||||
## Verification After Remediation
|
||||
|
||||
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
|
||||
changed-package race tests, `go test ./...`, `go test -race ./...`,
|
||||
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
|
||||
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||
`git diff --check` all passed. Existing replay and cancellation regression
|
||||
tests remain intact; the HTTP adapter required no changes.
|
||||
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
@@ -0,0 +1,183 @@
|
||||
# Resumable Speaker Transcription and Replayable Uploads
|
||||
|
||||
Status: Implemented.
|
||||
|
||||
## Goal and User Intent
|
||||
|
||||
Preserve completed speaker transcriptions when another speaker fails, so an
|
||||
ordinary run or resume does not repeat expensive successful work. Separately,
|
||||
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
|
||||
recoverable connection or stream failure.
|
||||
|
||||
The motivating incident exhausted four transcription attempts for one speaker
|
||||
after three other speakers had completed. The final error reported an HTTP/2
|
||||
peer `PROTOCOL_ERROR` and an inability to replay the request body without
|
||||
`Request.GetBody`.
|
||||
|
||||
These are related reliability improvements, not a promise to eliminate all
|
||||
service failures. This document records the target state and scope. The
|
||||
completed implementation sequence is retained in Git history; current behavior
|
||||
is documented in the canonical guides linked below.
|
||||
|
||||
## Pre-implementation Limitations
|
||||
|
||||
The [transcribe stage](../internal/stage-transcribe.md) writes successful
|
||||
responses into the invocation's run directory. It installs canonical transcripts
|
||||
and returns its per-file metadata only after every speaker succeeds. A failure
|
||||
therefore leaves no durable per-speaker progress that ordinary resume can use,
|
||||
even though some completed JSON files may remain on disk.
|
||||
|
||||
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
|
||||
the application level, reopening audio for each attempt. Its multipart request
|
||||
body is streamed through a pipe and has no replay callback. Transport-level
|
||||
recovery within an attempt cannot reconstruct a consumed body. Providing that
|
||||
capability addresses the replay limitation, not the underlying peer protocol
|
||||
error; persistent failures can still require service or proxy investigation.
|
||||
|
||||
## Targeted End State
|
||||
|
||||
### Durable Per-Speaker Completion
|
||||
|
||||
Each successfully transcribed speaker file is validated, saved, and recorded
|
||||
durably before that work is treated as reusable. The session manifest remains
|
||||
the progress authority. Follow the existing analyze-stage checkpoint pattern
|
||||
without turning speakers into pipeline stages or introducing a general task
|
||||
engine.
|
||||
|
||||
Ordinary execution reconciles the current prepared audio set with recorded
|
||||
speaker results. It reuses valid completed work and schedules failed,
|
||||
unattempted, missing, or obsolete results. A result is not reusable merely
|
||||
because a matching filename exists. Evidence binds it to the speaker/audio
|
||||
identity, audio content, Narratio-visible recognition settings, and the saved
|
||||
transcript's integrity.
|
||||
|
||||
The relevant recognition settings remain the WhisperX service URL and language.
|
||||
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
|
||||
remain operational settings rather than reasons to discard completed results.
|
||||
Changes to private service models or configuration remain an operator reason
|
||||
to force transcription; Narratio does not inspect the service's private state.
|
||||
|
||||
Changes to individual audio inputs require replacement of affected results,
|
||||
not automatic retranscription of unrelated unchanged speakers. New speakers
|
||||
need transcription; removed speakers do not remain in the current aggregate
|
||||
output set. Records must describe the inputs actually used, not grant reuse to
|
||||
old bytes under a changed identity.
|
||||
|
||||
### Aggregate Completion, Failure, and Force
|
||||
|
||||
The overall stage succeeds only when every currently required speaker has a
|
||||
validated, durably recorded result. Incomplete per-speaker progress does not
|
||||
make the aggregate stage successful or authorize downstream consumption of an
|
||||
incomplete collection. Existing dependent-stage invalidation remains in effect.
|
||||
|
||||
Retain bounded concurrency and the current fail-fast behavior: a speaker
|
||||
failure cancels outstanding work. Results already checkpointed remain reusable
|
||||
after failure, cancellation, or process interruption. A later ordinary run or
|
||||
resume retries the remaining work; it does not need a new CLI command.
|
||||
|
||||
Checkpoint writes must be serialized so concurrently finishing speakers cannot
|
||||
overwrite each other's progress. A failed persistence operation must not
|
||||
advertise an unrecorded result as complete. Previously committed progress remains
|
||||
the recovery authority, and checkpoint errors must be surfaced rather than
|
||||
reported as successful transcription.
|
||||
|
||||
`--force` on transcribe regenerates all currently required speakers instead of
|
||||
reusing their prior completions. Successful replacements are checkpointed too,
|
||||
so an ordinary retry after a failed forced invocation can retain the newly
|
||||
completed work. Old results for speakers not successfully regenerated must not
|
||||
satisfy that unfinished forced replacement on the next ordinary retry.
|
||||
|
||||
This deliberately relaxes the current all-speakers-before-canonical-installation
|
||||
implementation: individual durable results may exist while the stage remains
|
||||
failed or incomplete. The all-speakers-before-stage-success boundary remains.
|
||||
|
||||
### Replayable WhisperX Requests
|
||||
|
||||
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
|
||||
same request. Replay reopens the audio and reproduces the multipart boundary,
|
||||
filename, fields, and framing used by the original request. Independent bodies
|
||||
must not share a consumed reader or mutable read position.
|
||||
|
||||
Preserve bounded-memory streaming rather than buffering entire audio files in
|
||||
memory. Correctly close every original and replay body and its audio resources;
|
||||
preserve source-read errors and cancellation, including partial uploads and
|
||||
early server responses. Successful response handling must not accidentally wait
|
||||
on, or report errors solely from, an abandoned body instead of the replay that
|
||||
produced the response.
|
||||
|
||||
Keep the existing application-level retry count, delay, classification, and
|
||||
per-attempt timeout semantics. Transport replays occur within an application
|
||||
attempt and its context; they are not additional Narratio-configured attempts.
|
||||
Do not add another custom retry loop, silently increase retries, or disable
|
||||
HTTP/2 globally as a substitute for replay support.
|
||||
|
||||
The adapter still validates and installs only successful JSON responses.
|
||||
Request replay does not provide exactly-once server execution: a lost response
|
||||
can cause repeated transcription work, as can existing application retries.
|
||||
No new service-side idempotency protocol is required by this feature.
|
||||
|
||||
## Required Change Areas
|
||||
|
||||
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
|
||||
evidence, with safe checkpoint updates and preservation through runner failure
|
||||
handling. Review manifest validation, copying, serialization, and existing
|
||||
publish/restore handling wherever the new records pass through those paths.
|
||||
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
|
||||
checkpoint validated completions, rebuild aggregate outputs from the current
|
||||
set, and account for partial progress in ordinary and forced execution.
|
||||
Review successful-stage reuse as well as failed-stage retries so stale audio
|
||||
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
|
||||
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
|
||||
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
|
||||
directory surviving cleanup. Do not introduce a second authoritative ledger
|
||||
or infer completion by scanning abandoned runs.
|
||||
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
|
||||
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
|
||||
an upstream service change.
|
||||
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
|
||||
checkpoint failure; changed/missing inputs or outputs; force followed by an
|
||||
ordinary retry; concurrent successful completions; and aggregate downstream
|
||||
gating. Extend existing fixtures rather than duplicating analyze's complete
|
||||
test suite. Exercise actual body reconstruction after partial consumption,
|
||||
multipart equivalence, cancellation/resource cleanup, and existing bounded
|
||||
streaming and retry behavior—not just the presence of a callback. Default
|
||||
tests remain offline and use local/fake HTTP boundaries, never paid services.
|
||||
- **Documentation:** Update the canonical
|
||||
[stage guide](../internal/stage-transcribe.md),
|
||||
[manifest guide](../internal/manifest.md),
|
||||
[WhisperX contract](../integrations/whisperx.md),
|
||||
[operations guide](../operations.md), and
|
||||
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
|
||||
partial completion, ordinary retry versus force, and the distinction between
|
||||
request replay limitations and persistent peer protocol failures.
|
||||
|
||||
## Compatibility and Scope Boundaries
|
||||
|
||||
Existing speaker identities, raw transcript formats, and the fixed pipeline
|
||||
order remain unchanged. Existing manifests remain readable, but must not be
|
||||
treated as containing per-speaker evidence they never recorded. Recovery relies
|
||||
on verified recorded evidence, not guessed associations between old transcripts
|
||||
and current audio. No general legacy-migration framework is part of the feature.
|
||||
|
||||
Automatic adoption of unrecorded outputs from previously failed runs is outside
|
||||
this feature. Such files may be recoverable after verifying their inputs, but
|
||||
their presence alone does not prove completion. No recovery/import command,
|
||||
speaker-selection CLI, historical invocation ledger, new service configuration
|
||||
discovery, or broader orchestration refactor is included.
|
||||
|
||||
The feature does not diagnose the remote service/proxy's underlying HTTP/2
|
||||
problem, guarantee recovery from persistent protocol errors, or change the
|
||||
policy for continuing unrelated work after failure.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
|
||||
completed, ordinary resume retains A and B, retries C, and completes D. No
|
||||
downstream stage receives an incomplete successful transcription collection.
|
||||
Force replaces all four, while an ordinary retry can reuse checkpoints from
|
||||
that forced replacement attempt.
|
||||
|
||||
The HTTP transport can reconstruct the upload after consuming the original
|
||||
body, without whole-file memory buffering, lost source errors, or stranded
|
||||
upload resources. A persistent peer error still fails normally after the
|
||||
configured application attempts, without discarding checkpointed speakers.
|
||||
960
docs/troubleshooting.md
Normal file
960
docs/troubleshooting.md
Normal file
@@ -0,0 +1,960 @@
|
||||
# Troubleshooting
|
||||
|
||||
Operational diagnosis guide for common Narratio failures.
|
||||
|
||||
## Config file not found
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails to resolve `pipeline.yml`, `campaign.yml`, or `session.yml`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing files in default search paths;
|
||||
- wrong campaign selection;
|
||||
- omitted explicit flags.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||
|
||||
Relevant reference: [Configuration discovery](./config.md#discovery-and-selection).
|
||||
|
||||
## Session template placeholders rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- load error says session file must be concrete or contains `{{ ... }}` placeholders.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- using template content as runtime session config.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04 --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- generate concrete session YAML with `narratio session init`.
|
||||
|
||||
Relevant reference: [Operations: Session Initialization](./operations.md#session-initialization).
|
||||
|
||||
## Strict decode or schema validation failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown field / invalid value error during config load.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- stale field name, typo, invalid enum, invalid duration/path format, or an
|
||||
unsupported local manifest schema/state marker.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- align config with [Configuration](./config.md) and the
|
||||
[maintained examples](../examples/README.md).
|
||||
- for a manifest schema error, move the unsupported local state aside and
|
||||
regenerate it; restore a verified current committed snapshot or republish
|
||||
healthy regenerated state when appropriate. Do not add version or stage
|
||||
markers by hand.
|
||||
- if an older manifest is rejected for its retired cleanup-obligation field,
|
||||
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, and use
|
||||
the cleanup stage. Do not treat manual JSON editing as the normal migration
|
||||
path.
|
||||
|
||||
Relevant reference: [Configuration](./config.md).
|
||||
|
||||
## Party schema version rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- configuration loading reports that `party.schema_version` is required or
|
||||
unsupported.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the campaign `party.yml` is an unversioned legacy document, or names a
|
||||
schema other than `narratio.party.v1`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- replace the file with one exact, single-document
|
||||
[`narratio.party.v1` party](./integrations/party.md); do not add a version
|
||||
marker to an otherwise incompatible legacy structure. The versioned party is
|
||||
the only configured roster authority.
|
||||
|
||||
Relevant reference: [Canonical Party Input](./integrations/party.md).
|
||||
|
||||
## Unexpected imported or profile value
|
||||
|
||||
Symptom:
|
||||
|
||||
- an effective configuration value differs from the root file, or a duplicate
|
||||
ownership/configuration error is hard to locate.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio config sources --config /path/pipeline.yml --profile testing
|
||||
```
|
||||
|
||||
Add `--campaign` or `--campaign-file` when the pipeline has party-driven
|
||||
artifact families. The output identifies each effective logical field's root,
|
||||
import, profile, default, campaign, party, or family source without printing
|
||||
the field value or credential contents.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- move a duplicated base field so it has one owner;
|
||||
- correct the selected profile or its overlay; or
|
||||
- correct the campaign party/family declaration that owns generated values.
|
||||
|
||||
To review what would actually change before switching profiles, run `config
|
||||
diff` with the same pipeline and campaign selectors. It compares normalized
|
||||
effective values rather than YAML formatting or source-file layout.
|
||||
|
||||
Relevant reference: [Configuration inspection](./config.md#read-only-effective-pipeline-inspection).
|
||||
|
||||
## Audio mode conflict
|
||||
|
||||
Symptom:
|
||||
|
||||
- validation fails on session audio configuration.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- configured both local and S3 session audio inputs.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
Relevant reference: [Session configuration](./config.md#session).
|
||||
|
||||
## `--artifacts` selection error
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown artifact key or invalid `--artifacts` usage.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- key not defined in `pipeline.scriptorium.artifacts`;
|
||||
- empty list entry (for example trailing comma);
|
||||
- `run-stage` used with non-`analyze`/`publish` target.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- provide only configured keys and use `--artifacts` with supported commands/stages.
|
||||
|
||||
Relevant reference: [CLI artifact selection](./cli.md).
|
||||
|
||||
## Bounded run prerequisite is unusable
|
||||
|
||||
Symptom:
|
||||
|
||||
- `run` or `session plan` reports that a prerequisite stage is absent or has a
|
||||
pending, running, failed, stale, or interrupted status before the selected
|
||||
start.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- `--from` excludes upstream work that has not reached the terminal
|
||||
`succeeded` or `skipped` state in the session manifest.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session plan 2026-04-04 --from render --through analyze
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- widen the bounded range to include the first reported stage, or recover that
|
||||
stage explicitly with `run-stage` before retrying. The failed check does not
|
||||
create a run-local directory or modify the manifest. Narratio does not
|
||||
resume-validate excluded prefix stages, and stages after `--through` are not
|
||||
prerequisites.
|
||||
|
||||
If prerequisite statuses are terminal but a selected stage reports a missing,
|
||||
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
||||
that owns it; do not edit the manifest to bypass the selected stage's concrete
|
||||
input validation.
|
||||
|
||||
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
||||
|
||||
## Transcription retries speakers that previously finished
|
||||
|
||||
Symptom:
|
||||
|
||||
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
|
||||
finish earlier; or
|
||||
- an older successful session retranscribes all speakers once after upgrade.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the earlier response or canonical file was not followed by a successful
|
||||
speaker manifest checkpoint;
|
||||
- the prepared audio bytes/path, WhisperX URL, or language changed;
|
||||
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
|
||||
no longer matches its recorded size/checksum;
|
||||
- the previous invocation used `--force`, which durably cleared pre-force
|
||||
speaker authority; or
|
||||
- the manifest has only legacy aggregate transcribe state and cannot bind old
|
||||
outputs to their source audio.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- inspect the reported speaker and filesystem error, repair the prepared input
|
||||
or canonical-output path when needed, then retry transcribe normally;
|
||||
- allow a legacy aggregate-only record to regenerate once; and
|
||||
- do not edit the manifest or copy an old JSON file into place as a substitute
|
||||
for a successful checkpoint.
|
||||
|
||||
```bash
|
||||
narratio run-stage transcribe 2026-04-04
|
||||
```
|
||||
|
||||
Valid earlier speaker checkpoints remain reusable even while aggregate
|
||||
transcribe status is failed or interrupted. Merge remains gated until
|
||||
transcribe reaches aggregate success with the complete current set.
|
||||
|
||||
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
|
||||
|
||||
## WhisperX model changed but transcription was reused
|
||||
|
||||
Symptom:
|
||||
|
||||
- WhisperX model weights, prompts, or private server configuration changed at
|
||||
the same configured URL, but Narratio still considers speaker transcripts
|
||||
current.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio can fingerprint its configured URL and language, not private state
|
||||
loaded behind the service boundary.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- explicitly force transcribe to establish a durable reset, then use ordinary
|
||||
retry if the replacement attempt is interrupted.
|
||||
|
||||
```bash
|
||||
narratio run-stage transcribe 2026-04-04 --force
|
||||
```
|
||||
|
||||
Transport replay and application retries do not provide an exactly-once remote
|
||||
execution guarantee. Investigate server-side failures or duplicate work using
|
||||
WhisperX diagnostics; do not infer completion from a server log without the
|
||||
local speaker checkpoint.
|
||||
|
||||
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
|
||||
[Transcribe internals](./internal/stage-transcribe.md).
|
||||
|
||||
## Transcription manifest checkpoint failed
|
||||
|
||||
Symptom:
|
||||
|
||||
- transcribe reports that a speaker completed but the session manifest could
|
||||
not be saved.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the workspace is unwritable or out of space;
|
||||
- the manifest path or parent directory is unsafe or the wrong type; or
|
||||
- a transient filesystem error interrupted atomic replacement.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported filesystem or permission problem and retry transcribe
|
||||
normally. Narratio reconciles from the last readable collection and reuses
|
||||
only successfully checkpointed speakers.
|
||||
|
||||
A canonical raw transcript left by the failed checkpoint is incidental and
|
||||
does not become reusable authority. Preserve the manifest and diagnostics if
|
||||
the failure repeats; do not advertise the file by editing manifest evidence.
|
||||
|
||||
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
|
||||
and [Transcribe internals](./internal/stage-transcribe.md).
|
||||
|
||||
## Notarius credential reference unavailable
|
||||
|
||||
Symptom:
|
||||
|
||||
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
|
||||
name for `pipeline.notarius.llm_api_key`, including during inspection or a
|
||||
run that would not execute extraction.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the named environment variable is absent or empty;
|
||||
- the selected secrets directory lacks a file with that exact name;
|
||||
- a process environment variable exists but is empty, so bootstrap correctly
|
||||
leaves it in place; or
|
||||
- the Narratio reference name does not match `api_key_env` in the selected
|
||||
Notarius/PromptKit profile.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- set the exact nonempty environment variable or provide an equivalently named
|
||||
file through the conventional or explicit secrets directory;
|
||||
- remove or set the field to `null` when that deployment needs no credential;
|
||||
and
|
||||
- use `config show`, `config sources`, and `config diff` to inspect the public
|
||||
reference name and ownership without exposing its value.
|
||||
|
||||
Narratio resolves selected credential references during ordinary pipeline
|
||||
loading, before stage selection or resume. Credential changes alone do not
|
||||
invalidate a completed extraction, but the current reference must still resolve
|
||||
for the command to proceed.
|
||||
|
||||
Relevant references: [Secrets](./config.md#secrets-handling) and
|
||||
[Notarius credential](./config.md#notarius-credential).
|
||||
|
||||
## Notarius executable missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails while resolving or starting the Notarius executable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||
- a configured executable path is wrong.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- install a compatible Notarius release or correct the binary setting, then
|
||||
rerun extraction.
|
||||
|
||||
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius exits nonzero
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports a Notarius exit error instead of a receipt.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||
the receipt and is not merged with diagnostics.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||
|
||||
After a failed replacement, an older immutable bundle may still exist even
|
||||
though the current session manifest has no successful extraction payload. This
|
||||
is expected audit state, not a signal to relink the old bundle manually.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Prepared Notarius reference missing or inconsistent
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction or resume validation reports that a configured reference source is
|
||||
unavailable, unsafe, empty, or checksum-inconsistent and recommends
|
||||
`prepare --force`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `prepare` has not run since the campaign/session stable input changed;
|
||||
- the configured source file is missing;
|
||||
- a prepared `inputs/` file or its manifest record was modified independently;
|
||||
- a spell-catalog binding exists without an effective `spell_catalog_file`.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the campaign/session input path, then refresh canonical prepared
|
||||
evidence before extraction:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Do not point Notarius directly at the original source path or edit the manifest
|
||||
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
||||
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius reference snapshot cache conflict
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports that a path beneath
|
||||
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
|
||||
non-regular or unsafe entry.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- stop commands for the affected session and inspect the exact reported path;
|
||||
- investigate unexpected links or special files as possible tampering; and
|
||||
- remove only the reported conflicting leaf or digest directory after
|
||||
confirming it belongs to the reconstructible reference cache.
|
||||
|
||||
A missing snapshot or corrupt regular snapshot is reconstructed automatically
|
||||
from currently verified prepared input. Do not recursively delete the session
|
||||
root, edit the manifest checksum, or remove cache entries while extraction is
|
||||
active.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius resume executes work instead of reusing it
|
||||
|
||||
Symptom:
|
||||
|
||||
- Narratio executes `extract`, and Notarius makes provider calls despite
|
||||
`pipeline.notarius.resume: true`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- checkpoint recording is disabled in the Notarius configuration;
|
||||
- Notarius is using a nonpersistent or different checkpoint root;
|
||||
- transcript/reference bytes, reference paths, pipeline/profile content,
|
||||
local-backend endpoint, selected lanes, or another checkpoint identity input
|
||||
changed; or
|
||||
- Narratio extraction was forced, which deliberately omits `--resume`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- validate the deployed Notarius configuration and configure explicit
|
||||
persistent chunk-plan and checkpoint roots;
|
||||
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
|
||||
and
|
||||
- retry without force only when reuse is intended.
|
||||
|
||||
Remember that Narratio skipping a completed `extract` stage is separate from
|
||||
Notarius reusing work inside an extraction that does execute. Narratio does not
|
||||
read the Notarius configuration to infer checkpoint enablement.
|
||||
|
||||
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius reference selector or generated-handoff collision
|
||||
|
||||
Symptom:
|
||||
|
||||
- Notarius exits nonzero with an undeclared reference-slot, incompatible media,
|
||||
or external/generated reference collision error.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- a selector does not identify a slot declared by the selected Notarius target;
|
||||
- a prepared file does not satisfy that slot's Notarius media contract; or
|
||||
- a CLI binding attempts to replace a same-run generated D&D handoff.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- compare external bindings with the selected Notarius pipeline's canonical
|
||||
consumer documentation;
|
||||
- keep only campaign-owned external slots on the CLI; and
|
||||
- leave registry, scene, combat, and occurrence handoffs to Notarius pipeline
|
||||
composition.
|
||||
|
||||
Narratio validates selector structure and prepared evidence, while Notarius
|
||||
owns slot declarations, media compatibility, and generated-handoff conflicts.
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||
lane descriptor, or payload path even though Notarius exited successfully.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- Narratio and Notarius versions disagree on their consumer contract;
|
||||
- the configured pipeline or lane constraints are stale;
|
||||
- output paths escape the bundle or traverse symlinks.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- compare installed Notarius output with the canonical Notarius contracts,
|
||||
including receipt `index_file: index.json` and index management names
|
||||
`manifest.json`, `rejected.json`, `warnings.json`, and `diagnostics.json`; align
|
||||
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||
checks.
|
||||
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Required Notarius lane rejected or missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||
or incompatible, including after a zero exit.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||
- correct the Notarius module or the exact declared lane contract;
|
||||
- remove an output declaration only if downstream consumers genuinely no longer
|
||||
require that source, then rerun extraction.
|
||||
|
||||
Every configured output is required. Narratio does not promote a partial result.
|
||||
|
||||
## Extraction resume invalidated
|
||||
|
||||
Symptom:
|
||||
|
||||
- a previously successful extraction runs again during ordinary continuation.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the pipeline ID or configured output contracts changed;
|
||||
- a configured prepared reference selector, source, path, checksum, or byte
|
||||
size changed;
|
||||
- the direct trimmed transcript identity or bytes changed; or
|
||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||
checksum no longer validates.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
||||
and investigate it rather than replacing files manually.
|
||||
|
||||
Executable, config path, timeout, working directory, and Notarius credential
|
||||
changes are operational and do not by themselves invalidate reusable extraction
|
||||
evidence. A newly selected credential must nevertheless resolve during command
|
||||
configuration loading.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
Symptom:
|
||||
|
||||
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||
but Narratio still considers the previous extraction resumable.
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio fingerprints its invocation contract and prepared Narratio reference
|
||||
identities, not the contents of other transitive Notarius inputs. Always force
|
||||
extraction after changing those external inputs; downstream
|
||||
successful stages are then marked stale normally.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Analysis artifact evidence is not current
|
||||
|
||||
Symptom:
|
||||
|
||||
- ordinary continuation or `session plan` schedules one or more configured
|
||||
artifacts even though a canonical output file exists; or
|
||||
- publish reports a configured artifact source unavailable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the per-artifact record is stale, missing, failed, unselected, or malformed;
|
||||
- a configured prompt/profile, dependency, input identity, output path, or
|
||||
effective variable changed; or
|
||||
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
|
||||
longer matches its manifest evidence.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session artifacts 2026-04-04
|
||||
narratio session plan 2026-04-04 --from analyze --through analyze
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- investigate unexpected path or checksum changes as possible tampering;
|
||||
- otherwise let the selected analyze work rerun, or explicitly regenerate only
|
||||
the affected targets; and
|
||||
- never edit the fingerprint/checksum in the manifest or copy an old file into
|
||||
the canonical path as a substitute for current evidence.
|
||||
|
||||
```bash
|
||||
narratio analyze 2026-04-04 --artifacts session_recap
|
||||
```
|
||||
|
||||
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
|
||||
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
|
||||
|
||||
## Analysis manifest checkpoint failed
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze reports that it could not save the session manifest after completing
|
||||
an artifact.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the workspace is unwritable or out of space;
|
||||
- the manifest path or its parent directory has unsafe permissions or type; or
|
||||
- a transient local filesystem error interrupted atomic manifest replacement.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported filesystem or permission problem;
|
||||
- inspect `session artifacts` to identify the last durable current records; and
|
||||
- rerun analyze normally. Narratio reuses earlier successful checkpoints and
|
||||
retries the failed or non-current artifact and its required dependents.
|
||||
|
||||
Do not treat a newly created canonical artifact file as current when its
|
||||
checkpoint failed, and do not edit the manifest to advertise it. If the error
|
||||
recurs, preserve the manifest and run-local diagnostics before repairing the
|
||||
workspace.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#persistence-semantics).
|
||||
|
||||
## Built-in transcript or bounds source is unavailable
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze, extract, render, or publish says a built-in source is unavailable
|
||||
even though a file exists at its usual session path.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the producing stage has not succeeded, its output lacks the exact registered
|
||||
source, run ID, or checksum, its evidence is duplicated or stale, or the
|
||||
durable bytes no longer match the manifest checksum.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- do not copy or rename the file to make it discoverable. Rerun the producing
|
||||
stage to regenerate both the bytes and current manifest evidence, then retry
|
||||
the consumer.
|
||||
|
||||
## An older manifest schema is rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- Narratio rejects a local session manifest because `schema_version` is missing
|
||||
or unsupported; or
|
||||
- restore rejects a committed remote session manifest for the same reason.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the manifest predates the supported canonical state contract. Narratio
|
||||
rejects it at the schema boundary rather than interpreting aggregate output
|
||||
lists or incidental files as current authority; or
|
||||
- an older strict-decoding Narratio binary is reading a current manifest that
|
||||
contains additive transcribe speaker state it does not recognize.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- for unsupported local state, move the old manifest aside and regenerate, or
|
||||
restore a verified current committed snapshot when one is authoritative;
|
||||
- for unsupported committed remote state, regenerate healthy local state as
|
||||
needed and republish the session; and
|
||||
- for a downgrade rejection, return to the current binary or restore a verified
|
||||
pre-upgrade backup. Do not delete only the speaker fields by hand.
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
After current records exist, inspect them and publish explicitly. Do not copy
|
||||
old files into canonical paths merely to influence selection; availability is
|
||||
manifest-owned.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
|
||||
|
||||
## Scriptorium private input changed without a rerun
|
||||
|
||||
Symptom:
|
||||
|
||||
- a prompt, profile, imported configuration file, executable, or other input
|
||||
loaded privately by Scriptorium changed, but Narratio still considers an
|
||||
artifact current.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- analysis fingerprints cover Narratio-observable semantic identities, not
|
||||
executable contents or arbitrary files and transitive configuration that
|
||||
Scriptorium loads behind its configured paths and identifiers.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- explicitly force the affected target after changing an unobserved private
|
||||
input. Force applies to explicit targets; current prerequisites remain
|
||||
reusable unless selected themselves.
|
||||
|
||||
```bash
|
||||
narratio analyze 2026-04-04 --artifacts session_recap
|
||||
```
|
||||
|
||||
Relevant reference: [Analyze Internals](./internal/stage-analyze.md#invariants).
|
||||
|
||||
## Previous-session artifact input missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare/analyze fails due to missing required previous-session artifact cache input.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing `session.previous_session_id`;
|
||||
- the selected previous-session commit does not declare the required canonical
|
||||
source as a `published_output`; or
|
||||
- the declared published object is unavailable or fails identity validation.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
or rerun prepare after correcting session config:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Session lock conflict (`.lock`)
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails acquiring session lock.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- another process is running for the same session;
|
||||
- a process still holds the operating-system lock while it is shutting down.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
||||
ps aux | grep narratio
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- wait for active process completion;
|
||||
- retry after an interrupted holder has exited; the kernel releases its lock
|
||||
even though the `.lock` metadata file remains for inspection.
|
||||
|
||||
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
|
||||
|
||||
## Restore conflict without `--force`
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore fails with conflict count.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- local durable files differ from remote restore sources.
|
||||
- a restore target or one of its ancestors is a symlink or another non-regular
|
||||
filesystem entry.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite an eligible
|
||||
regular local file; repair a symlink or other non-regular conflict first.
|
||||
|
||||
If the filesystem changed after the plan was printed, rerun restore after
|
||||
repairing the destination; installation performs the same safety check again.
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Restore current-state discovery failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore cannot find current pointer or current manifest.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- no committed publish current state;
|
||||
- storage credentials or connectivity failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- resolve storage/auth issue;
|
||||
- republish from healthy local state if current pointer is missing.
|
||||
|
||||
Relevant reference: [Operations: Publish Workflow](./operations.md#publish-workflow).
|
||||
|
||||
## Publish output failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- publish fails on missing required source, upload error, or commit write.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- required source file not produced;
|
||||
- lock/state expectations mismatch;
|
||||
- remote storage failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04 --remote
|
||||
narratio session status 2026-04-04
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- regenerate missing sources by rerunning prerequisite stages;
|
||||
- correct publish source/destination rules;
|
||||
- retry after storage failure is resolved.
|
||||
|
||||
Relevant reference: [Publish configuration](./config.md#publish-configuration-summary).
|
||||
|
||||
## Render markdown source missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze or publish fails because `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` is unavailable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- render stage was not executed after transcript changes;
|
||||
- render stage failed before producing canonical markdown outputs.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- rerun render and then retry downstream stage(s):
|
||||
|
||||
```bash
|
||||
narratio run-stage render 2026-04-04 --force
|
||||
narratio run-stage analyze 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Stage Execution](./operations.md#stage-execution-and-continuation-behavior).
|
||||
|
||||
## Secrets or storage credential failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- object-store command fails at initialization/auth.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing, unreadable, or invalid explicit `--secrets-dir`;
|
||||
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
|
||||
- missing credential environment variables;
|
||||
- invalid S3 endpoint/bucket settings.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -la /path/to/secrets_dir
|
||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
Relevant reference: [Secrets](./config.md#secrets-handling).
|
||||
|
||||
## S3 audio prepare failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare fails listing/downloading session S3 audio.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- incorrect `session.inputs.audio_s3.prefix`;
|
||||
- no matching `.flac` objects;
|
||||
- storage connectivity or permissions failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- verify prefix contents and storage access;
|
||||
- keep session audio mode consistent.
|
||||
|
||||
Relevant reference: [Operations](./operations.md).
|
||||
|
||||
## References
|
||||
|
||||
- [docs/cli.md](./cli.md)
|
||||
- [docs/config.md](./config.md)
|
||||
- [docs/operations.md](./operations.md)
|
||||
- [docs/internal/stage-publish.md](./internal/stage-publish.md)
|
||||
79
examples/README.md
Normal file
79
examples/README.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# Maintained Examples
|
||||
|
||||
These files are safe, copyable starting points for Narratio configuration and
|
||||
input structure. Replace placeholder identifiers, storage names, integration
|
||||
URLs, and paths for the target environment. Field meanings and defaults belong
|
||||
in the [configuration reference](../docs/config.md).
|
||||
|
||||
## Pipeline Configuration
|
||||
|
||||
- [Minimal pipeline](pipeline.minimal.yml): campaign discovery plus the required
|
||||
WhisperX URL.
|
||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
|
||||
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
|
||||
one family per `artifact-families/<key>.yml`, and the fixed
|
||||
`profiles/testing.yml` overlay. Production values live in the base. The
|
||||
bundle demonstrates an input-only artifact dependency, two character families
|
||||
sharing the campaign party, and a testing-only disabled artifact. Inspect it
|
||||
offline with:
|
||||
|
||||
```sh
|
||||
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
```
|
||||
|
||||
These inspection commands load configuration and resolve explicit credential
|
||||
references, but do not load a session or create runtime state. This bundle is
|
||||
intentionally local-storage and credential-reference free so the walkthrough
|
||||
needs no tools or cloud services.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||
|
||||
The existing `internal/config` example test loads and validates each pipeline
|
||||
with the sample campaign and a compatible local- or S3-audio session.
|
||||
|
||||
## Campaign And Session Configuration
|
||||
|
||||
- [Sample campaign](campaigns/sample-campaign/campaign.yml), its
|
||||
[session template](campaigns/sample-campaign/session.template.yml), and its
|
||||
adjacent stable inputs provide a complete campaign directory shape.
|
||||
- [Local-audio session](session.local-audio.yml) and
|
||||
[S3-audio session](session.s3-audio.yml) are concrete session files.
|
||||
- [Session template](session.template.yml) and the campaign-local equivalent
|
||||
demonstrate the narrow placeholder syntax consumed by `session init`; they
|
||||
are templates, not runtime session files.
|
||||
|
||||
## Input Fixtures
|
||||
|
||||
- [Speakers](speakers.yml), [autocorrect](autocorrect.yml), and
|
||||
[glossary](glossary.yml) show the standalone input shapes.
|
||||
- The sample campaign references its local
|
||||
[speakers](campaigns/sample-campaign/speakers.yml),
|
||||
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
|
||||
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
||||
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||
follows the Notarius v0.7 contract. Narratio derives the players projection
|
||||
from this party source.
|
||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
The examples contain environment-variable names but no credential values. Set
|
||||
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
|
||||
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
|
||||
examples that reference them. They use fictional campaign content and reserved
|
||||
example domains.
|
||||
|
||||
For a deployed bundle, keep credential files out of source control in a
|
||||
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
|
||||
environment-variable name, or select another directory with `--secrets-dir`.
|
||||
Static S3 credentials are optional: omit both S3 credential references to use
|
||||
the AWS SDK discovery chain.
|
||||
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
campaign_id: sample-campaign
|
||||
session_template_file: ./session.template.yml
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
party_file: ./party.yml
|
||||
spell_catalog_file: ./spell_catalog.json
|
||||
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
26
examples/campaigns/sample-campaign/party.yml
Normal file
26
examples/campaigns/sample-campaign/party.yml
Normal file
@@ -0,0 +1,26 @@
|
||||
schema_version: narratio.party.v1
|
||||
|
||||
characters:
|
||||
arannis:
|
||||
player:
|
||||
name: Rowan Hale
|
||||
character:
|
||||
name: Arannis
|
||||
alias:
|
||||
- Ari
|
||||
- The Grey Owl
|
||||
classes:
|
||||
- name: wizard
|
||||
level: 8
|
||||
brenna:
|
||||
player:
|
||||
name: Rowan Hale
|
||||
character:
|
||||
name: Brenna
|
||||
alias:
|
||||
- Shield of Dawn
|
||||
classes:
|
||||
- name: paladin
|
||||
level: 6
|
||||
- name: warlock
|
||||
level: 2
|
||||
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
match:
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
18
examples/campaigns/sample-campaign/spell_catalog.json
Normal file
18
examples/campaigns/sample-campaign/spell_catalog.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||
"catalogs": [
|
||||
{
|
||||
"id": "narratio.sample-campaign",
|
||||
"ruleset": "dnd-5e-2014",
|
||||
"source": {
|
||||
"title": "Narratio sample campaign spell names"
|
||||
},
|
||||
"spells": [
|
||||
{
|
||||
"name": "Aegis of Emberfall",
|
||||
"aliases": ["Emberfall Aegis"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
59
examples/pipeline.extraction-subset.yml
Normal file
59
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,59 @@
|
||||
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||
# only the three Notarius lanes it needs.
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
notarius:
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
references:
|
||||
glossary: narratio.input.glossary
|
||||
party: narratio.input.party
|
||||
players: narratio.input.players
|
||||
spell_catalog: narratio.input.spell_catalog
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
artifacts:
|
||||
session_brief:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_brief
|
||||
output_path: artifacts/session_brief.md
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.npc_registry
|
||||
required: true
|
||||
locations:
|
||||
source: narratio.extraction.location_registry
|
||||
required: true
|
||||
scenes:
|
||||
source: narratio.extraction.scene_descriptions
|
||||
required: true
|
||||
283
examples/pipeline.full.annotated.yml
Normal file
283
examples/pipeline.full.annotated.yml
Normal file
@@ -0,0 +1,283 @@
|
||||
# Full annotated pipeline example for implemented Narratio config fields.
|
||||
# Values are safe placeholders and must be adapted per environment.
|
||||
|
||||
workspace:
|
||||
# Optional: defaults to /var/lib/narratio.
|
||||
root: /var/lib/narratio/workspace
|
||||
# Optional: remove run-scoped workdir after successful publish commit.
|
||||
cleanup_after_publish: false
|
||||
|
||||
# Credential files, when used, belong in a sibling secrets/ directory or in the
|
||||
# directory selected with --secrets-dir; they are not pipeline configuration.
|
||||
|
||||
storage:
|
||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||
backend: s3
|
||||
s3:
|
||||
# Required when using S3 audio or S3 publish uploads.
|
||||
bucket: my-dnd-archive
|
||||
# Optional; defaults to "dnd".
|
||||
root_prefix: dnd
|
||||
# Optional region/endpoint settings.
|
||||
region: us-east-1
|
||||
endpoint: ""
|
||||
force_path_style: false
|
||||
# Optional; defaults shown explicitly.
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
# Optional command default when --campaign is omitted.
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
# Optional; defaults to /var/spool/narratio.
|
||||
root: /var/spool/narratio
|
||||
# Optional cleanup of run-scoped spool audio after successful publish commit.
|
||||
delete_audio_after_publish: false
|
||||
|
||||
publish:
|
||||
# Optional booleans; defaults are true.
|
||||
enabled: true
|
||||
upload_run: true
|
||||
# Optional publish output rules; sources use Narratio artifact source IDs.
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||
# are never implicit publish sources.
|
||||
- source: narratio.extraction.npc_registry
|
||||
dest: artifacts/extraction/npc-registry.json
|
||||
required: true
|
||||
|
||||
whisperx:
|
||||
# Required.
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
language: en
|
||||
timeout: 30m
|
||||
retries: 3
|
||||
retry_delay: 2s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
env:
|
||||
# Optional advanced tuning; set only when needed.
|
||||
overlap_word_run_gap: 1.0
|
||||
overlap_word_run_reorder_window: 1.0
|
||||
backchannel_max_duration: 2.0
|
||||
filler_max_duration: 1.25
|
||||
|
||||
audita:
|
||||
# Optional overrides; defaults shown explicitly where applicable.
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
base_url: ""
|
||||
model: ""
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_model: ""
|
||||
validation_llm_concurrency: 1
|
||||
transcript_description: ""
|
||||
config_path: /usr/local/etc/audita/config.yml
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
# Optional; defaults shown explicitly.
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
trim:
|
||||
# Optional; defaults shown explicitly.
|
||||
enabled: true
|
||||
output_path: transcripts/final.trimmed.json
|
||||
bounds:
|
||||
prompt_id: dnd.session_bounds
|
||||
profile_id: ""
|
||||
transcript_input_name: transcript
|
||||
output_path: artifacts/session_bounds.json
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
notarius:
|
||||
# Optional structured extraction after render and before analyze.
|
||||
enabled: true
|
||||
binary: notarius
|
||||
# Optional environment-variable reference. Match this name to api_key_env in
|
||||
# the selected Notarius/PromptKit profile; never put the credential here.
|
||||
llm_api_key: NOTARIUS_LLM_API_KEY
|
||||
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
|
||||
# is independent of Narratio's selected profile and may use a different name.
|
||||
# See docs/config.md and docs/integrations/notarius.md for the contracts.
|
||||
# config_profile: downstream-economy
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
working_directory: /usr/local/etc/notarius
|
||||
# Reuse compatible accepted Notarius checkpoints during executed extraction.
|
||||
# The Notarius config must enable recording and provide persistent cache roots.
|
||||
resume: true
|
||||
# auto uses or creates a source-addressed chunk plan. Forced Narratio
|
||||
# extraction overrides this with refresh and disables checkpoint loading.
|
||||
chunk_cache: auto
|
||||
# External campaign references use prepared Narratio source IDs. Omit an
|
||||
# optional binding when the selected Notarius pipeline does not need it.
|
||||
references:
|
||||
glossary: narratio.input.glossary
|
||||
party: narratio.input.party
|
||||
players: narratio.input.players
|
||||
spell_catalog: narratio.input.spell_catalog
|
||||
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||
outputs:
|
||||
item_registry:
|
||||
lane_id: item-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/item-registry
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
item_occurrences:
|
||||
lane_id: item-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/item-occurrences
|
||||
spells:
|
||||
lane_id: spells
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.spells
|
||||
schema_version: v1
|
||||
module_key: dnd/spells
|
||||
combat_turns:
|
||||
lane_id: combat-turns
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.combat_turns
|
||||
schema_version: v1
|
||||
module_key: dnd/combat-turns
|
||||
npc_occurrences:
|
||||
lane_id: npc-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-occurrences
|
||||
location_occurrences:
|
||||
lane_id: location-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/location-occurrences
|
||||
enemy_events:
|
||||
lane_id: enemy-events
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.enemy_events
|
||||
schema_version: v1
|
||||
module_key: dnd/enemy-events
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
# Configured artifact keys map to source IDs narratio.artifact.<key>.
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
players:
|
||||
source: narratio.input.players
|
||||
required: true
|
||||
party:
|
||||
source: narratio.input.party
|
||||
required: true
|
||||
glossary:
|
||||
source: narratio.input.glossary
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
|
||||
# Example dependent artifact:
|
||||
# - depends_on entries use artifact keys.
|
||||
# - narratio.artifact.<key> sources require matching depends_on membership.
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
campaign_name: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
# No delivery provider is currently implemented.
|
||||
mode: noop
|
||||
@@ -1,114 +1,6 @@
|
||||
workspace:
|
||||
root: ./tmp/narratio-workspace
|
||||
|
||||
storage:
|
||||
backend: local
|
||||
|
||||
secrets:
|
||||
# Optional: load environment variables from files in this directory.
|
||||
# File name = env var name; file contents = env var value.
|
||||
env_dir: /var/local/narratio/secrets
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
language: "en"
|
||||
timeout: "30m"
|
||||
retries: 3
|
||||
retry_delay: "2s"
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
binary: "seriatim"
|
||||
timeout: "10m"
|
||||
output_schema: "seriatim-intermediate"
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
env:
|
||||
overlap_word_run_gap: 1.0
|
||||
overlap_word_run_reorder_window: 1.0
|
||||
backchannel_max_duration: 2.0
|
||||
filler_max_duration: 1.25
|
||||
|
||||
audita:
|
||||
binary: "audita"
|
||||
timeout: "3h"
|
||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||
modules:
|
||||
- glossary
|
||||
- homophones
|
||||
- glossary
|
||||
- spoken_word
|
||||
- grammar
|
||||
- homophones
|
||||
- glossary
|
||||
base_url: "https://openrouter.ai/api/v1"
|
||||
model: "openrouter/google/gemma-4-31b-it"
|
||||
llm_concurrency: 1
|
||||
validation_model: ""
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
# Session-workdir-relative when not absolute.
|
||||
output_path: "transcripts/normalized.json"
|
||||
output_schema: "seriatim-intermediate"
|
||||
report: true
|
||||
|
||||
trim:
|
||||
enabled: true
|
||||
# Session-workdir-relative when not absolute.
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
# Empty means use prompt default profile.
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
scriptorium:
|
||||
binary: "scriptorium"
|
||||
config_path: "/etc/scriptorium/config.yml"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
profile_id: "local-quality"
|
||||
output_path: "artifacts/session_recap.md"
|
||||
timeout: "10m"
|
||||
# Optional per-artifact override of global scriptorium.render_debug.
|
||||
# render_debug: true
|
||||
inputs:
|
||||
transcript:
|
||||
# Available transcript sources:
|
||||
# - trimmed_transcript (recommended for session_recap)
|
||||
# - normalized_transcript (recommended for future full-session analysis)
|
||||
# - processed_transcript (raw Audita-polished output)
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
# Optional: set when previous recap is available.
|
||||
path: ""
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
artifacts:
|
||||
output_dir: artifacts
|
||||
|
||||
notification:
|
||||
timeout: 10s
|
||||
|
||||
128
examples/pipeline.production.yml
Normal file
128
examples/pipeline.production.yml
Normal file
@@ -0,0 +1,128 @@
|
||||
workspace:
|
||||
root: /var/lib/narratio/workspace
|
||||
cleanup_after_publish: true
|
||||
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: my-dnd-archive
|
||||
root_prefix: dnd
|
||||
region: us-east-1
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
root: /var/spool/narratio
|
||||
delete_audio_after_publish: true
|
||||
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
- source: narratio.transcript.final_trimmed_markdown
|
||||
dest: transcripts/final.trimmed.md
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
language: en
|
||||
timeout: 45m
|
||||
retries: 3
|
||||
retry_delay: 3s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
|
||||
audita:
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
players:
|
||||
source: narratio.input.players
|
||||
required: true
|
||||
party:
|
||||
source: narratio.input.party
|
||||
required: true
|
||||
glossary:
|
||||
source: narratio.input.glossary
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
mode: noop
|
||||
@@ -0,0 +1,20 @@
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_items:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_items
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||
inputs:
|
||||
character_meta:
|
||||
source: narratio.member_artifact.character_meta
|
||||
required: true
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
aliases: character.alias_summary
|
||||
publish:
|
||||
enabled: true
|
||||
required: false
|
||||
dest_pattern: artifacts/characters/{character_id}/items.md
|
||||
@@ -0,0 +1,13 @@
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_meta
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
player_name: player.name
|
||||
class_summary: character.class_summary
|
||||
11
examples/production-testing/artifacts/player_handout.yml
Normal file
11
examples/production-testing/artifacts/player_handout.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
player_handout:
|
||||
enabled: true
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/player-handout.md
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
11
examples/production-testing/artifacts/session_recap.yml
Normal file
11
examples/production-testing/artifacts/session_recap.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/session-recap.md
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
12
examples/production-testing/conf.d/platform.yml
Normal file
12
examples/production-testing/conf.d/platform.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
workspace:
|
||||
root: ./workspace
|
||||
|
||||
campaigns:
|
||||
root: ../../campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
cache:
|
||||
root: ./cache
|
||||
|
||||
spool:
|
||||
root: ./spool
|
||||
7
examples/production-testing/conf.d/publish.yml
Normal file
7
examples/production-testing/conf.d/publish.yml
Normal file
@@ -0,0 +1,7 @@
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: false
|
||||
outputs:
|
||||
- source: narratio.transcript.final_markdown
|
||||
dest: transcripts/final.md
|
||||
required: true
|
||||
2
examples/production-testing/conf.d/storage.yml
Normal file
2
examples/production-testing/conf.d/storage.yml
Normal file
@@ -0,0 +1,2 @@
|
||||
storage:
|
||||
backend: local
|
||||
18
examples/production-testing/conf.d/transcript.yml
Normal file
18
examples/production-testing/conf.d/transcript.yml
Normal file
@@ -0,0 +1,18 @@
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
language: en
|
||||
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
output_schema: seriatim-intermediate
|
||||
|
||||
audita:
|
||||
binary: audita
|
||||
modules: [glossary, grammar]
|
||||
output_schema: audita-v1
|
||||
model: narratio-production-model-placeholder
|
||||
validation_model: narratio-production-validator-placeholder
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
14
examples/production-testing/pipeline.yml
Normal file
14
examples/production-testing/pipeline.yml
Normal file
@@ -0,0 +1,14 @@
|
||||
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
|
||||
# these explicit path and glob imports; Narratio never scans directories itself.
|
||||
configkit:
|
||||
imports:
|
||||
- path: conf.d/platform.yml
|
||||
- path: conf.d/storage.yml
|
||||
- path: conf.d/transcript.yml
|
||||
- path: conf.d/publish.yml
|
||||
- glob: artifacts/*.yml
|
||||
- glob: artifact-families/*.yml
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: ./scriptorium/config.yml
|
||||
18
examples/production-testing/profiles/testing.yml
Normal file
18
examples/production-testing/profiles/testing.yml
Normal file
@@ -0,0 +1,18 @@
|
||||
audita:
|
||||
model: narratio-testing-model-placeholder
|
||||
validation_model: narratio-testing-validator-placeholder
|
||||
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
profile_id: testing-placeholder
|
||||
player_handout:
|
||||
profile_id: testing-placeholder
|
||||
testing_notes:
|
||||
enabled: false
|
||||
output_path: artifacts/testing-notes.md
|
||||
artifact_families:
|
||||
character_meta:
|
||||
profile_id: testing-placeholder
|
||||
character_items:
|
||||
profile_id: testing-placeholder
|
||||
5
examples/session.local-audio.yml
Normal file
5
examples/session.local-audio.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
@@ -1,10 +0,0 @@
|
||||
session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
|
||||
6
examples/session.s3-audio.yml
Normal file
6
examples/session.s3-audio.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
3
examples/session.template.yml
Normal file
3
examples/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
@@ -1,5 +1,5 @@
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
|
||||
32
go.mod
32
go.mod
@@ -1,5 +1,33 @@
|
||||
module gitea.maximumdirect.net/eric/narratio
|
||||
|
||||
go 1.25.0
|
||||
go 1.26.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/google/go-cmp v0.7.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
)
|
||||
|
||||
46
go.sum
46
go.sum
@@ -1,3 +1,49 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.1 h1:etfTI/dxO3NnS18La0V6IeUh/Iv+3PjVfEyaN8hTuOM=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// NoopRunner is a deterministic no-op analyzer adapter.
|
||||
type NoopRunner struct{}
|
||||
|
||||
// Run returns the requested output path with placeholder metadata.
|
||||
func (n *NoopRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
return AnalyzeResult{ArtifactPath: req.OutputPath, Metadata: map[string]any{"placeholder": true}}, nil
|
||||
}
|
||||
|
||||
// FakeRunner captures analyze requests and returns deterministic responses.
|
||||
type FakeRunner struct {
|
||||
Requests []AnalyzeRequest
|
||||
Err error
|
||||
Result AnalyzeResult
|
||||
}
|
||||
|
||||
// Run records request and returns configured response.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return AnalyzeResult{}, f.Err
|
||||
}
|
||||
res := f.Result
|
||||
if res.ArtifactPath == "" {
|
||||
res.ArtifactPath = req.OutputPath
|
||||
}
|
||||
if res.Metadata == nil {
|
||||
res.Metadata = map[string]any{"fake": true}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
fake := &FakeRunner{}
|
||||
req := AnalyzeRequest{ArtifactType: "session-log", OutputPath: "artifacts/session-log.md"}
|
||||
|
||||
res, err := fake.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 || fake.Requests[0].ArtifactType != "session-log" {
|
||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
||||
}
|
||||
if res.ArtifactPath != req.OutputPath {
|
||||
t.Fatalf("artifact path = %q, want %q", res.ArtifactPath, req.OutputPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerError(t *testing.T) {
|
||||
fake := &FakeRunner{Err: errors.New("boom")}
|
||||
_, err := fake.Run(context.Background(), AnalyzeRequest{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Package analyzer declares the adapter contract for artifact analysis generation.
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// TODO: implement analyzer integration once the analyzer contract is finalized.
|
||||
|
||||
// Runner is the adapter boundary for analyzer invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error)
|
||||
}
|
||||
|
||||
// AnalyzeRequest describes one analyzer artifact generation request.
|
||||
type AnalyzeRequest struct {
|
||||
ArtifactType string
|
||||
ProcessedTranscriptPath string
|
||||
ContextReferences []string
|
||||
OutputPath string
|
||||
GeneratedConfigPath string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// AnalyzeResult describes analyzer output.
|
||||
type AnalyzeResult struct {
|
||||
ArtifactPath string
|
||||
Metadata map[string]any
|
||||
}
|
||||
@@ -4,10 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// NoopRunner is a deterministic no-op audita adapter.
|
||||
@@ -84,17 +83,17 @@ func materializePlaceholders(req PolishRequest) error {
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -117,14 +116,14 @@ func writeJSONIfRequested(path string, payload any) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
|
||||
}
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||
}
|
||||
if err := subprocess.WriteFileAtomic(path, data, 0o644); err != nil {
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "audita.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "audita.stderr.log"),
|
||||
}
|
||||
|
||||
19
internal/adapters/audita/generated_config.go
Normal file
19
internal/adapters/audita/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package audita
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO: implement a real Audita subprocess/service adapter.
|
||||
|
||||
// Runner is the adapter boundary for audita polish invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
|
||||
@@ -15,19 +13,15 @@ type Runner interface {
|
||||
|
||||
// PolishRequest describes an audita invocation.
|
||||
type PolishRequest struct {
|
||||
GeneratedConfigPath string
|
||||
MergedTranscriptPath string
|
||||
OutputProcessedPath string
|
||||
GlossaryPath string
|
||||
ReportPath string
|
||||
WorkDir string
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
ValidationModel string
|
||||
ValidationLLMConcurrency *int
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
GeneratedConfigPath string
|
||||
MergedTranscriptPath string
|
||||
OutputProcessedPath string
|
||||
GlossaryPath string
|
||||
ReportPath string
|
||||
WorkDir string
|
||||
Modules []string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// PolishResult describes a polish output.
|
||||
|
||||
@@ -5,23 +5,35 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
|
||||
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
|
||||
|
||||
// MaxReportOutputBytes bounds Audita's optional report JSON result.
|
||||
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
||||
|
||||
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
|
||||
type SubprocessRunnerConfig struct {
|
||||
Binary string
|
||||
Timeout time.Duration
|
||||
LLMAPIKeyEnv string
|
||||
LLMAPIKey string
|
||||
LLMAPIKeyReference string
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
LLMConcurrency *int
|
||||
TranscriptDescription string
|
||||
ConfigPath string
|
||||
OutputSchema string
|
||||
WorkDirRetention string
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
ValidationModel string
|
||||
ValidationLLMConcurrency *int
|
||||
Report bool
|
||||
@@ -31,11 +43,17 @@ type SubprocessRunnerConfig struct {
|
||||
type SubprocessRunner struct {
|
||||
binary string
|
||||
timeout time.Duration
|
||||
llmAPIKeyEnv string
|
||||
llmAPIKey string
|
||||
llmAPIKeyReference string
|
||||
modules []string
|
||||
baseURL string
|
||||
model string
|
||||
llmConcurrency *int
|
||||
transcriptDescription string
|
||||
configPath string
|
||||
outputSchema string
|
||||
workDirRetention string
|
||||
totalLLMConcurrency *int
|
||||
proposalLLMConcurrency *int
|
||||
validationModel string
|
||||
validationLLMConcurrency *int
|
||||
report bool
|
||||
@@ -45,11 +63,17 @@ type SubprocessRunner struct {
|
||||
func NewSubprocessRunnerFromConfigValues(
|
||||
binary string,
|
||||
timeout string,
|
||||
llmAPIKeyEnv string,
|
||||
llmAPIKey string,
|
||||
llmAPIKeyReference string,
|
||||
modules []string,
|
||||
baseURL string,
|
||||
model string,
|
||||
llmConcurrency *int,
|
||||
transcriptDescription string,
|
||||
configPath string,
|
||||
outputSchema string,
|
||||
workDirRetention string,
|
||||
totalLLMConcurrency *int,
|
||||
proposalLLMConcurrency *int,
|
||||
validationModel string,
|
||||
validationLLMConcurrency *int,
|
||||
report bool,
|
||||
@@ -64,11 +88,17 @@ func NewSubprocessRunnerFromConfigValues(
|
||||
return NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: binary,
|
||||
Timeout: parsedTimeout,
|
||||
LLMAPIKeyEnv: llmAPIKeyEnv,
|
||||
LLMAPIKey: llmAPIKey,
|
||||
LLMAPIKeyReference: llmAPIKeyReference,
|
||||
Modules: modules,
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
LLMConcurrency: llmConcurrency,
|
||||
TranscriptDescription: transcriptDescription,
|
||||
ConfigPath: configPath,
|
||||
OutputSchema: outputSchema,
|
||||
WorkDirRetention: workDirRetention,
|
||||
TotalLLMConcurrency: totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: proposalLLMConcurrency,
|
||||
ValidationModel: validationModel,
|
||||
ValidationLLMConcurrency: validationLLMConcurrency,
|
||||
Report: report,
|
||||
@@ -83,33 +113,42 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
if cfg.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||
}
|
||||
if len(cfg.Modules) == 0 {
|
||||
return nil, fmt.Errorf("audita modules must include at least one module")
|
||||
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
|
||||
return nil, fmt.Errorf("audita credential value and reference must be provided together")
|
||||
}
|
||||
for i, module := range cfg.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(cfg.BaseURL) == "" {
|
||||
return nil, fmt.Errorf("audita base url is required")
|
||||
}
|
||||
u, err := url.Parse(cfg.BaseURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||
u, err := url.Parse(cfg.BaseURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
||||
}
|
||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||
}
|
||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return nil, fmt.Errorf("audita model is required")
|
||||
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
|
||||
}
|
||||
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
|
||||
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
||||
}
|
||||
switch strings.TrimSpace(cfg.OutputSchema) {
|
||||
case "", "bare-segments", "audita-v1":
|
||||
default:
|
||||
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
|
||||
}
|
||||
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
||||
case "", "always", "auto", "never":
|
||||
default:
|
||||
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
|
||||
}
|
||||
|
||||
modules := make([]string, len(cfg.Modules))
|
||||
for i, m := range cfg.Modules {
|
||||
@@ -119,11 +158,17 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
return &SubprocessRunner{
|
||||
binary: strings.TrimSpace(cfg.Binary),
|
||||
timeout: cfg.Timeout,
|
||||
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
|
||||
llmAPIKey: cfg.LLMAPIKey,
|
||||
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
|
||||
modules: modules,
|
||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
model: strings.TrimSpace(cfg.Model),
|
||||
llmConcurrency: cfg.LLMConcurrency,
|
||||
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
|
||||
configPath: strings.TrimSpace(cfg.ConfigPath),
|
||||
outputSchema: strings.TrimSpace(cfg.OutputSchema),
|
||||
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
|
||||
totalLLMConcurrency: cfg.TotalLLMConcurrency,
|
||||
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
|
||||
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
||||
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
||||
report: cfg.Report,
|
||||
@@ -152,41 +197,31 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
|
||||
reqModules := req.Modules
|
||||
if len(reqModules) == 0 {
|
||||
if reqModules == nil {
|
||||
reqModules = append([]string(nil), r.modules...)
|
||||
}
|
||||
args := r.buildArgs(req, reqModules)
|
||||
|
||||
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
|
||||
credentialPresent := false
|
||||
credentialPresent := r.llmAPIKeyReference != ""
|
||||
env := map[string]string{}
|
||||
if credentialEnvVar != "" {
|
||||
credential, ok := os.LookupEnv(credentialEnvVar)
|
||||
if !ok || strings.TrimSpace(credential) == "" {
|
||||
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
|
||||
}
|
||||
env["AUDITA_LLM_API_KEY"] = credential
|
||||
credentialPresent = true
|
||||
}
|
||||
primaryConcurrencyViaEnv := false
|
||||
if r.llmConcurrency != nil {
|
||||
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
|
||||
primaryConcurrencyViaEnv = true
|
||||
if credentialPresent {
|
||||
env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
|
||||
}
|
||||
|
||||
if req.GeneratedConfigPath != "" {
|
||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
|
||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
|
||||
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
DiagnosticOwner: "audita",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
wrappedMessage := fmt.Sprintf(
|
||||
@@ -196,7 +231,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
req.StderrLogPath,
|
||||
)
|
||||
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
|
||||
"%s: %w",
|
||||
wrappedMessage,
|
||||
err,
|
||||
@@ -204,11 +239,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
|
||||
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||
}
|
||||
if r.report {
|
||||
if err := validateJSONFile(req.ReportPath); err != nil {
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,21 +258,25 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: r.binary,
|
||||
Metadata: map[string]any{
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": reqModules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": reqModules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
|
||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
|
||||
return PolishResult{
|
||||
ProcessedTranscriptPath: req.OutputProcessedPath,
|
||||
ReportPath: req.ReportPath,
|
||||
@@ -249,16 +288,20 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: r.binary,
|
||||
Metadata: map[string]any{
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -269,14 +312,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
||||
req.MergedTranscriptPath,
|
||||
"--glossary", req.GlossaryPath,
|
||||
"--output", req.OutputProcessedPath,
|
||||
"--modules", strings.Join(modules, ","),
|
||||
"--base-url", r.baseURL,
|
||||
"--model", r.model,
|
||||
"--work-dir", req.WorkDir,
|
||||
}
|
||||
if r.baseURL != "" {
|
||||
args = append(args, "--base-url", r.baseURL)
|
||||
}
|
||||
if r.model != "" {
|
||||
args = append(args, "--model", r.model)
|
||||
}
|
||||
if len(modules) > 0 {
|
||||
args = append(args, "--modules", strings.Join(modules, ","))
|
||||
}
|
||||
if r.report {
|
||||
args = append(args, "--report-json", req.ReportPath)
|
||||
}
|
||||
if r.transcriptDescription != "" {
|
||||
args = append(args, "--transcript-description", r.transcriptDescription)
|
||||
}
|
||||
if r.configPath != "" {
|
||||
args = append(args, "--config", r.configPath)
|
||||
}
|
||||
if r.outputSchema != "" {
|
||||
args = append(args, "--output-schema", r.outputSchema)
|
||||
}
|
||||
if r.workDirRetention != "" {
|
||||
args = append(args, "--work-dir-retention", r.workDirRetention)
|
||||
}
|
||||
if r.totalLLMConcurrency != nil {
|
||||
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
|
||||
}
|
||||
if r.proposalLLMConcurrency != nil {
|
||||
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
|
||||
}
|
||||
if r.validationModel != "" {
|
||||
args = append(args, "--validation-model", r.validationModel)
|
||||
}
|
||||
@@ -286,37 +353,39 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
||||
return args
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
|
||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
|
||||
payload := map[string]any{
|
||||
"schema": "audita.generated.v1",
|
||||
"binary": r.binary,
|
||||
"args": args,
|
||||
"timeout": r.timeout.String(),
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"report_enabled": r.report,
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"glossary_path": req.GlossaryPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
"schema": "audita.generated.v1",
|
||||
"binary": r.binary,
|
||||
"args": args,
|
||||
"timeout": r.timeout.String(),
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"report_enabled": r.report,
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"glossary_path": req.GlossaryPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
if r.llmConcurrency != nil {
|
||||
payload["llm_concurrency"] = *r.llmConcurrency
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func validateProcessedOutput(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
@@ -345,9 +414,9 @@ func addSubprocessStreamHint(message string, runErr error) string {
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
@@ -355,3 +424,11 @@ func validateJSONFile(path string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadFileBounded(path, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
@@ -25,16 +21,23 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
wrapper := writeAuditaHelperWrapper(t)
|
||||
llmConcurrency := 1
|
||||
totalLLMConcurrency := 3
|
||||
proposalLLMConcurrency := 2
|
||||
validationLLMConcurrency := 2
|
||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: wrapper,
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary", "homophones", "glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
TranscriptDescription: "Campaign Session 42",
|
||||
ConfigPath: "/etc/audita/config.yml",
|
||||
OutputSchema: "audita-v1",
|
||||
WorkDirRetention: "auto",
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||
Report: true,
|
||||
@@ -42,13 +45,14 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||
}
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
ReportPath: filepath.Join(dir, "audita.report.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
@@ -93,11 +97,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
"process", req.MergedTranscriptPath,
|
||||
"--glossary", req.GlossaryPath,
|
||||
"--output", req.OutputProcessedPath,
|
||||
"--modules", "glossary,homophones,glossary",
|
||||
"--work-dir", req.WorkDir,
|
||||
"--base-url", "https://openrouter.ai/api/v1",
|
||||
"--model", "openrouter/google/gemma-4-31b-it",
|
||||
"--work-dir", req.WorkDir,
|
||||
"--modules", "glossary,homophones,glossary",
|
||||
"--report-json", req.ReportPath,
|
||||
"--transcript-description", "Campaign Session 42",
|
||||
"--config", "/etc/audita/config.yml",
|
||||
"--output-schema", "audita-v1",
|
||||
"--work-dir-retention", "auto",
|
||||
"--total-llm-concurrency", "3",
|
||||
"--proposal-llm-concurrency", "2",
|
||||
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
||||
"--validation-llm-concurrency", "2",
|
||||
}
|
||||
@@ -107,8 +117,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
||||
}
|
||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
|
||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
|
||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||
}
|
||||
|
||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||
@@ -120,51 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
|
||||
t.Fatalf("error = %q, want missing credential guidance", err.Error())
|
||||
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
@@ -181,10 +171,7 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||
@@ -192,13 +179,13 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKeyEnv: "",
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKey: "",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -206,30 +193,79 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
}
|
||||
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "inherited-from-parent" {
|
||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want inherited-from-parent", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
|
||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--modules" {
|
||||
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--base-url" {
|
||||
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
if rec.Args[i] == "--model" {
|
||||
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -245,27 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "super-secret") {
|
||||
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -278,24 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -308,24 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -338,24 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -368,24 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -398,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected binary validation error")
|
||||
}
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout parse error")
|
||||
}
|
||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKey: "synthetic-secret",
|
||||
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected base url validation error")
|
||||
@@ -516,7 +536,7 @@ func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunne
|
||||
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
merged := filepath.Join(dir, "merged.json")
|
||||
merged := filepath.Join(dir, "base.json")
|
||||
glossary := filepath.Join(dir, "glossary.yml")
|
||||
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, glossary, "terms: []\n")
|
||||
@@ -524,7 +544,7 @@ func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: merged,
|
||||
GlossaryPath: glossary,
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||
|
||||
28
internal/adapters/notarius/fake.go
Normal file
28
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package notarius
|
||||
|
||||
import "context"
|
||||
|
||||
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||
type FakeRunner struct {
|
||||
Requests []RunRequest
|
||||
Result RunResult
|
||||
Err error
|
||||
}
|
||||
|
||||
// Run records the request and returns the configured result or error.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
copyRequest := req
|
||||
if req.ConfigProfile != nil {
|
||||
profile := *req.ConfigProfile
|
||||
copyRequest.ConfigProfile = &profile
|
||||
}
|
||||
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
|
||||
f.Requests = append(f.Requests, copyRequest)
|
||||
if f.Err != nil {
|
||||
return RunResult{}, f.Err
|
||||
}
|
||||
return f.Result, nil
|
||||
}
|
||||
167
internal/adapters/notarius/runner.go
Normal file
167
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,167 @@
|
||||
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
||||
|
||||
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||
}
|
||||
|
||||
// ReferenceBinding maps one normalized Notarius selector to an absolute
|
||||
// external reference path.
|
||||
type ReferenceBinding struct {
|
||||
Selector string
|
||||
Path string
|
||||
}
|
||||
|
||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||
type RunRequest struct {
|
||||
Binary string
|
||||
// LLMAPIKey is an optional already-resolved credential reference.
|
||||
LLMAPIKey *configkit.Secret
|
||||
ConfigProfile *string
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
InputPath string
|
||||
OutputRoot string
|
||||
WorkingDirectory string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
Timeout time.Duration
|
||||
Resume bool
|
||||
ChunkCache string
|
||||
References []ReferenceBinding
|
||||
}
|
||||
|
||||
// Receipt is the transport-neutral successful run receipt.
|
||||
type Receipt struct {
|
||||
SchemaVersion string
|
||||
RunID string
|
||||
PipelineID string
|
||||
OutputDirectory string
|
||||
IndexFile string
|
||||
NormalizedOutputCount int
|
||||
RejectedOutputCount int
|
||||
WarningGroupCount int
|
||||
WarningOccurrenceCount int
|
||||
DiagnosticGroupCount int
|
||||
DiagnosticOccurrenceCount int
|
||||
DiagnosticsTruncated bool
|
||||
ValidationStatus string
|
||||
ValidationSummaries []ValidationSummary
|
||||
DebugDirectory string
|
||||
}
|
||||
|
||||
// ValidationSummary retains the bounded outcome of one Notarius producer result.
|
||||
type ValidationSummary struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ChunkID string
|
||||
Status string
|
||||
RejectingValidators []string
|
||||
ReasonCodes []string
|
||||
IncompleteValidators []string
|
||||
ProducerAttemptCount int
|
||||
TerminalAction string
|
||||
}
|
||||
|
||||
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||
type LaneDescriptor struct {
|
||||
LaneID string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
ModuleKey string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||
type PipelineDescriptor struct {
|
||||
ArtifactKind string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// Index describes the validated bundle-management and artifact paths.
|
||||
type Index struct {
|
||||
Path string
|
||||
ManifestFile string
|
||||
ManifestPath string
|
||||
RejectedFile string
|
||||
RejectedPath string
|
||||
WarningsFile string
|
||||
WarningsPath string
|
||||
DiagnosticsFile string
|
||||
DiagnosticsPath string
|
||||
Lanes []LaneDescriptor
|
||||
ChunkMap *PipelineDescriptor
|
||||
EvidenceContext *PipelineDescriptor
|
||||
}
|
||||
|
||||
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||
type RejectionSummary struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ChunkID string
|
||||
ValidatorName string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// WarningSummary retains structured warning identity without free-form messages.
|
||||
type WarningSummary struct {
|
||||
Disposition string
|
||||
Category string
|
||||
ReasonCode string
|
||||
Origin DiagnosticOrigin
|
||||
OccurrenceCount int
|
||||
}
|
||||
|
||||
// DiagnosticOrigin identifies the framework-owned pipeline location of a finding.
|
||||
type DiagnosticOrigin struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ValidatorKey string
|
||||
}
|
||||
|
||||
// DiagnosticSummary retains bounded advisory or observation group metadata.
|
||||
type DiagnosticSummary struct {
|
||||
Disposition string
|
||||
Category string
|
||||
ReasonCode string
|
||||
Origin DiagnosticOrigin
|
||||
OccurrenceCount int
|
||||
}
|
||||
|
||||
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||
type RunResult struct {
|
||||
Receipt Receipt
|
||||
Index Index
|
||||
BundleRoot string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
Rejections []RejectionSummary
|
||||
Warnings []WarningSummary
|
||||
Diagnostics []DiagnosticSummary
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user