314 Commits

Author SHA1 Message Date
792cbd0677 Prepare the v2.2.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-22 01:50:49 +00:00
5dca34e0e0 Allow checkpoints for empty audio accepted by WhisperX 2026-09-22 00:53:28 +00:00
54d335c820 Repair stale transcription outputs when resuming 2026-09-22 00:50:51 +00:00
96b4e73f53 Document resumable transcription and replay 2026-09-22 00:31:32 +00:00
6e6054f744 Verify WhisperX transport replay 2026-09-22 00:22:48 +00:00
dc3288d15a Make WhisperX uploads replayable 2026-09-22 00:15:48 +00:00
d872edca94 Integrate transcription checkpoints across lifecycle 2026-09-22 00:00:51 +00:00
2ae63dbc2b Checkpoint speaker transcriptions incrementally 2026-09-21 16:20:53 +00:00
26aa0cce29 Reconcile reusable speaker transcriptions 2026-09-21 16:09:45 +00:00
7b0bf99ed1 Add durable speaker transcription state 2026-09-21 16:01:19 +00:00
89ada1b269 Plan resumable transcription and replayable uploads 2026-09-21 15:50:48 +00:00
3438fe82c9 Document Notarius configuration profiles 2026-09-21 15:16:11 +00:00
a51a513804 Verify Notarius profile lifecycle behavior 2026-09-21 15:11:19 +00:00
995354f1a5 Track Notarius profiles in extraction reuse 2026-09-21 15:07:36 +00:00
931c9591cd Add Notarius configuration profile support 2026-09-21 15:03:29 +00:00
46096fc4ba Plan Notarius configuration profile support 2026-09-21 14:54:47 +00:00
d3ecd907ee Update ConfigKit and prepare Narratio v2.1.2
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-21 13:42:21 +00:00
442469b9f5 Fix derived publish destinations with explicit requiredness
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-21 03:06:13 +00:00
c45e042462 Prepare the v2.1.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-16 18:50:30 +00:00
207da1b6ef Remove completed Notarius cache roadmaps
Some checks failed
ci/woodpecker/push/verify Pipeline was canceled
2026-09-16 18:48:33 +00:00
9a1609435e Clarify Notarius compatibility requirements 2026-09-16 18:48:23 +00:00
13b83d7ee2 Fix Notarius snapshots in relative workspaces 2026-09-16 18:48:23 +00:00
a0312ed034 Document Notarius cache reuse workflow 2026-09-16 18:14:40 +00:00
8ff74e1251 Prove Notarius cache reuse end to end 2026-09-16 18:07:37 +00:00
1b480d4ed5 Verify Notarius cache lifecycle boundaries 2026-09-16 17:59:52 +00:00
e13d87a077 Use stable verified Notarius reference snapshots 2026-09-16 17:53:29 +00:00
5202f16972 Forward Notarius cache controls during extraction 2026-09-16 17:44:28 +00:00
5086596f94 Add typed Notarius cache controls 2026-09-16 17:42:16 +00:00
f7fc64db10 Plan Notarius cache reuse 2026-09-16 17:34:21 +00:00
6b0778b420 Clean up completed roadmap documentation
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
2026-09-16 16:39:37 +00:00
5b92b3e790 Document Notarius credential forwarding 2026-09-16 14:36:44 +00:00
cc43906ae9 Verify Notarius credential lifecycle 2026-09-16 14:31:54 +00:00
dcaa2ad9c6 Forward Notarius credentials securely 2026-09-16 14:27:06 +00:00
9d68cfcf2f Add Notarius credential configuration 2026-09-16 14:22:45 +00:00
2d346733d6 Plan Notarius credential forwarding 2026-09-16 14:14:56 +00:00
cd75bf6d5a Complete the ConfigKit v0.3.0 adoption 2026-09-16 04:33:19 +00:00
e08de98ae6 Upgrade ConfigKit to v0.3.0 2026-09-16 04:29:05 +00:00
df213bc2ba Isolate pipeline configuration discovery 2026-09-16 04:26:24 +00:00
869e68e834 Plan the ConfigKit upgrade and test isolation 2026-09-16 04:16:30 +00:00
ed4623bc41 Fix profile loading with ConfigKit history
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-07 13:10:29 +00:00
3ce198197d Report secrets bootstrap outcomes safely 2026-09-07 12:14:10 +00:00
0ac8072a9a Complete artifact provenance reporting 2026-09-07 12:08:19 +00:00
3ec54b8c72 Preserve null collections in effective YAML 2026-09-07 12:03:45 +00:00
47b3e5b55b Validate hidden base artifact ownership conflicts 2026-09-07 11:59:49 +00:00
99d1a0b926 Honor explicit Notarius working directories 2026-09-07 11:54:35 +00:00
884a62e8a6 Derive family member publish destinations 2026-09-07 11:52:57 +00:00
f7a40afc7a Complete the ConfigKit migration audit 2026-09-07 04:58:52 +00:00
a4f37ff427 Document the v2 operator migration 2026-09-07 04:55:13 +00:00
2b97ef3722 Add effective artifact catalog command 2026-09-07 04:46:20 +00:00
63a86c43f4 Infer artifact dependencies from configured inputs 2026-09-07 04:39:49 +00:00
92432232ef Resolve typed credentials safely 2026-09-07 04:29:00 +00:00
d31ba5a410 Bootstrap secrets before configuration loading 2026-09-07 04:14:04 +00:00
f0225e937c Migrate remaining configuration documents to ConfigKit 2026-09-07 04:02:21 +00:00
9e40c729e1 Adopt ConfigKit for pipeline loading 2026-09-07 03:54:41 +00:00
0e6ba414b3 Unify effective config projection 2026-09-07 03:35:46 +00:00
10a2e89965 Separate config normalization from validation 2026-09-07 03:32:50 +00:00
f8e3e651e6 Reduce supported platform matrix 2026-09-07 03:28:50 +00:00
1558c9ccae Plan ConfigKit adoption and platform simplification 2026-09-07 03:21:06 +00:00
b68435128b Document the v1.7.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-05 19:24:35 +00:00
5fb454a714 Remove completed roadmap and audit documents 2026-09-05 19:18:48 +00:00
79cf9f1139 Remove unused dependency wrappers 2026-09-05 18:49:16 +00:00
9bb203476d Carry dependencies through config commands 2026-09-05 18:48:25 +00:00
0726786016 Close quality and documentation workstream 2026-09-05 18:16:15 +00:00
204caf9d24 Improve structural test diagnostics 2026-09-05 18:08:49 +00:00
85663f5bc9 Replace restore phase globals 2026-09-05 18:05:18 +00:00
f51b9bb046 Carry dependencies through operator commands 2026-09-05 18:00:50 +00:00
dd905c4daa Carry dependencies through pipeline commands 2026-09-05 17:53:56 +00:00
c3307ab5fb Replace configuration and secret loading globals 2026-09-05 17:48:22 +00:00
1484f51f48 Plan final refactoring workstream 2026-09-05 17:38:57 +00:00
51d21676c1 Create run workspaces only when needed 2026-09-05 16:34:59 +00:00
c2806a1615 Complete cleanup stage workstream 2026-09-05 15:42:00 +00:00
078f661e3a Remove retired cleanup manifest state 2026-09-05 15:30:31 +00:00
adc60aa588 Integrate cleanup into pipeline execution 2026-09-05 15:27:08 +00:00
8d9eb5cc40 Implement idempotent cleanup behavior 2026-09-05 15:15:28 +00:00
563acce012 Define cleanup authorization and targets 2026-09-05 15:09:57 +00:00
49c359feb8 Plan cleanup stage refactor 2026-09-05 14:53:00 +00:00
38202d4472 Correct roadmap verification command 2026-09-05 14:17:43 +00:00
6d07dafc9f Remove unused lock dispatcher 2026-09-05 14:17:22 +00:00
6c66e18c4b Show carried output locations 2026-09-05 14:16:20 +00:00
4048996652 Validate published output declarations 2026-09-05 14:14:33 +00:00
470450e509 Complete remote publication workstream 2026-09-05 13:24:54 +00:00
7e78c22fec Remove retired lock authorities 2026-09-05 13:17:46 +00:00
610eaefb27 Migrate lock commands to independent objects 2026-09-05 13:11:11 +00:00
4bc33773fa Integrate artifact locks into publish 2026-09-05 13:04:08 +00:00
b74e31e1cc Model publish output outcomes 2026-09-05 12:52:55 +00:00
2f1e4407db Separate publish transfer and commit mechanics 2026-09-05 12:48:12 +00:00
4ccb4a73ca Separate publish source and output planning 2026-09-05 12:43:27 +00:00
145baac393 Permit safe cross-run output declarations 2026-09-05 12:37:58 +00:00
be59bff690 Define independent remote lock objects 2026-09-05 12:27:51 +00:00
14b4fee214 Add idempotent object deletion 2026-09-05 12:22:44 +00:00
7f84fc5d1e Plan remote publication refactoring 2026-09-05 11:57:22 +00:00
a8b47f6185 Strengthen Notarius determinism tests 2026-09-05 03:38:07 +00:00
5b62a8d60d Make cross-configuration validation deterministic 2026-09-05 03:37:00 +00:00
0d4d71b9dc Complete configuration diagnostics workstream 2026-09-05 00:42:47 +00:00
979d8049b1 Clarify pipeline composition envelope handling 2026-09-05 00:31:32 +00:00
abd405015b Make Scriptorium validation deterministic 2026-09-05 00:29:05 +00:00
fddf78dc80 Make Notarius validation transactional 2026-09-05 00:26:54 +00:00
3136c26d4e Separate configuration domain validators 2026-09-05 00:24:58 +00:00
4b84b4223d Separate configuration validation ownership 2026-09-05 00:21:04 +00:00
7a0e2cb11b Plan configuration refactoring 2026-09-04 23:37:02 +00:00
74ab9218ff Close analyze checkpointing documentation 2026-09-04 18:38:11 +00:00
da99857fd0 Reconcile removed analysis artifacts 2026-09-04 18:37:17 +00:00
a520cfdcc8 Close analyze checkpointing workstream 2026-09-04 16:16:19 +00:00
7febccf550 Complete analyze checkpoint recovery 2026-09-04 16:10:54 +00:00
74977b4064 Checkpoint analyze artifacts incrementally 2026-09-04 16:04:59 +00:00
172e2d6a4f Consolidate analyze session execution state 2026-09-04 15:57:54 +00:00
5c520dc0d4 Decompose analyze artifact execution 2026-09-04 15:50:51 +00:00
6d18a3126e Plan analyze artifact checkpointing 2026-09-04 13:21:27 +00:00
1e222a4db3 Test real planning and execution parity 2026-09-04 13:03:39 +00:00
6a15052d77 Isolate runner object store construction 2026-09-04 13:01:00 +00:00
2e69c14898 Complete runner lifecycle refactor 2026-09-04 01:21:37 +00:00
dbda2a0161 Isolate runner finalization 2026-09-04 01:15:07 +00:00
2ee64f2eb1 Extract single-stage runner execution 2026-09-04 01:14:09 +00:00
8afe9427ca Isolate invocation collaborator composition 2026-09-04 01:11:11 +00:00
e0f19833a1 Establish explicit runner lifecycle state 2026-09-04 01:05:44 +00:00
ca1b694aa9 Unify pre-execution stage decisions 2026-09-04 01:01:46 +00:00
9331373245 Plan runner lifecycle refactor 2026-09-04 00:26:04 +00:00
ec072dc163 Finish analyze projection cleanup 2026-09-04 00:09:31 +00:00
f7ce1dcd56 Simplify previous manifest collection 2026-09-04 00:08:58 +00:00
df275874e3 Confine published diagnostic files 2026-09-04 00:07:50 +00:00
1da34ab5cd Complete session-only manifest migration 2026-09-03 04:22:19 +00:00
c0fbc14f2d Delete run manifest domain model 2026-09-03 04:16:36 +00:00
2d20900cbf Remove invocation analyze projection 2026-09-03 04:11:33 +00:00
7997f81f47 Make runner persistence session-only 2026-09-03 04:09:12 +00:00
6d748d65d3 Decouple publish from run manifests 2026-09-03 04:00:25 +00:00
49324d3f35 Plan session manifest simplification 2026-09-03 00:48:46 +00:00
c98b4ef4ef Align compatibility documentation 2026-09-02 14:38:51 +00:00
c8bae3a4a3 Clarify remote manifest recovery 2026-09-02 14:37:36 +00:00
ccbd7ee820 Require committed restore state 2026-09-02 14:36:02 +00:00
e9c4873030 Complete compatibility purge audit 2026-09-02 14:18:51 +00:00
57719b7d81 Require manifest-backed local artifacts 2026-09-02 14:09:32 +00:00
62b1f9b0ae Resolve previous artifacts from commit declarations 2026-09-02 13:53:23 +00:00
6c7a45d8d0 Require committed remote current state 2026-09-02 13:47:20 +00:00
82d75e7beb Remove legacy party configuration 2026-09-02 12:04:35 +00:00
61095e69ce Reject unversioned party documents 2026-09-02 11:57:51 +00:00
2cfca710ec Require explicit manifest schemas 2026-09-02 11:45:40 +00:00
9679117e94 Plan the compatibility purge 2026-09-02 03:53:36 +00:00
d67a0389e0 Cover bounded reads and cancelled replacements 2026-09-02 01:19:21 +00:00
9f84446509 Clean up failed Notarius extraction runs 2026-09-02 01:18:32 +00:00
ec9b0e919d Verify restored audio before installation 2026-09-02 01:17:21 +00:00
9bb7c1f4e3 Complete filesystem simplification workstream 2026-09-02 01:01:44 +00:00
3c092fbe34 Replace Notarius bundle promotion with rename 2026-09-02 00:56:53 +00:00
3a7e2e5175 Constrain restore installation to session root 2026-09-02 00:53:35 +00:00
45b27ad562 Protect restore planning from symlinks 2026-09-02 00:51:19 +00:00
1e418d6929 Simplify root-scoped cleanup 2026-09-02 00:48:40 +00:00
39b2665d8b Validate downloads before installing workspace files 2026-09-02 00:37:00 +00:00
ba858cc997 Migrate ordinary I/O from confined file helpers 2026-09-01 17:51:37 +00:00
0d57422b61 Simplify atomic file replacement and bounded reads 2026-09-01 17:40:08 +00:00
b7d989f61f Plan filesystem simplification 2026-09-01 15:34:39 +00:00
6813340f51 Strengthen transcription concurrency tests 2026-09-01 11:51:36 +00:00
eb788f17b8 Remove obsolete subprocess wait code 2026-09-01 11:48:34 +00:00
ebd3f415d3 Clarify Windows process setup failures 2026-09-01 11:48:00 +00:00
f15d8d8ce0 Preserve subprocess wait failures 2026-09-01 11:46:54 +00:00
6c7bfd5e31 Complete Go and process refactoring workstream 2026-08-31 23:56:39 +00:00
7c782a2603 Use errgroup for bounded transcription work 2026-08-31 23:55:08 +00:00
f1eef5b5e9 Move generated file writes into owning adapters 2026-08-31 23:53:21 +00:00
f14583321d Use stable Windows process handles for job assignment 2026-08-31 23:49:47 +00:00
04283e99d4 Separate subprocess execution lifecycle from policy 2026-08-31 23:48:49 +00:00
7284228bec Upgrade maintained Go baseline to 1.26 2026-08-31 23:45:54 +00:00
da74d7ad18 Plan Go and process refactoring 2026-08-31 23:40:42 +00:00
af5fbc1c4f Record maintainability audit results 2026-08-31 04:43:24 +00:00
5e4d2016e8 Record maintainability audit hypotheses 2026-08-31 04:30:03 +00:00
545aa6893b Prepare the v1.6.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-08-30 20:40:04 +00:00
98139f7e8b Harden release validation 2026-08-30 20:36:23 +00:00
f8fa0a2623 Document release procedure and complete release audit 2026-08-30 19:27:36 +00:00
9af773491b Use shared release scripts in Woodpecker 2026-08-30 19:22:26 +00:00
2a656f0f11 Add guarded release tag command 2026-08-30 19:20:12 +00:00
aec35a2d9b Add release candidate checker 2026-08-30 19:16:02 +00:00
23dc4e2078 Add release asset builder 2026-08-30 19:10:26 +00:00
c812fe3655 Harden pipeline state and plan release upgrades 2026-08-30 18:51:20 +00:00
3da97ca50c Add assembled pipeline configuration regression coverage 2026-08-30 15:26:23 +00:00
4c203d8588 Document pipeline configuration migration workflow 2026-08-30 15:19:30 +00:00
fcb5f825e1 Add split pipeline configuration example bundle 2026-08-30 15:17:46 +00:00
4c57ace2f6 Add semantic configuration profile comparison 2026-08-30 15:10:38 +00:00
dde7f76ecb Add configuration source reporting 2026-08-30 14:56:22 +00:00
a102db36af Add read-only configuration inspection commands 2026-08-30 14:43:25 +00:00
b5b1d22011 Expand family publish policies into concrete rules 2026-08-30 14:30:07 +00:00
c91599ef36 Add family artifact selection and provenance 2026-08-30 14:27:42 +00:00
257f10c9fb Resolve corresponding artifact family dependencies 2026-08-30 14:20:03 +00:00
fb9a4d14f4 Expand canonical party artifact families 2026-08-30 14:15:57 +00:00
c4435b76c4 Prepare canonical party and derived players inputs 2026-08-30 14:08:35 +00:00
61000a9466 Resolve canonical parties with campaign configuration 2026-08-30 14:01:51 +00:00
7e4ceb3d48 Add canonical party domain and players projection 2026-08-30 13:51:12 +00:00
4e991fa21d Expose pipeline profile selection and provenance 2026-08-30 13:41:26 +00:00
f86b17045d Unify application configuration loading 2026-08-30 13:32:45 +00:00
f302488075 Add named pipeline profile composition 2026-08-30 13:25:02 +00:00
8c1171478d Complete downstream semantic resume coverage 2026-08-30 13:14:33 +00:00
7ee637803d Protect transcript refinement resume semantics 2026-08-30 13:03:13 +00:00
4d6086fefb Protect initial pipeline stage resume semantics 2026-08-30 12:54:38 +00:00
82cb53e107 Add semantic configuration resume evidence 2026-08-30 12:44:26 +00:00
b97b12da7f Add explicit pipeline configuration imports 2026-08-30 12:33:43 +00:00
49ea747b17 Add presence-aware configuration composition 2026-08-30 12:21:27 +00:00
3b5e9db41f Plan pipeline configuration improvements 2026-08-30 03:41:56 +00:00
8b4b328c4e Prepare the v1.5.0 release notes
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-08-29 23:13:39 +00:00
ee2b8e63e6 Report the embedded release version 2026-08-29 23:13:01 +00:00
51edd384c0 Remove completed feature roadmaps 2026-08-29 23:11:19 +00:00
b804d0f2c8 Share analysis dependent staling traversal 2026-08-29 20:50:43 +00:00
fd5ccc668b Use one execution plan throughout the runner 2026-08-29 20:49:43 +00:00
0dc8ff9b52 Exclude external paths from analysis fingerprints 2026-08-29 20:45:27 +00:00
8657a28bdb Preserve flag-based artifact regeneration arguments 2026-08-29 20:43:44 +00:00
a9c5e4ad4e Recheck bounded prerequisites under lock 2026-08-29 20:42:51 +00:00
2176b4371d Close out the artifact workflow roadmap 2026-08-29 20:19:19 +00:00
effc10d75b Finalize artifact workflow documentation 2026-08-29 20:14:28 +00:00
5887839aa1 Add assembled workflow compatibility coverage 2026-08-29 20:08:48 +00:00
3128bef20a Add artifact-aware analyze resume planning 2026-08-29 20:01:57 +00:00
4e4e2b7d96 Preserve incremental analysis state on failure 2026-08-29 19:49:48 +00:00
99f4f9a0db Execute incremental analysis artifact plans 2026-08-29 19:43:28 +00:00
6abdd67bb5 Add incremental analysis work planning 2026-08-29 19:26:10 +00:00
c32e0c401f Add versioned analysis fingerprint reconciliation 2026-08-29 19:18:28 +00:00
ab5751459a Add deterministic analysis input identities 2026-08-29 19:02:16 +00:00
62de6abdbf Make configured artifacts manifest authoritative 2026-08-29 18:46:40 +00:00
903dc70682 Persist partial analysis artifact results 2026-08-29 18:36:21 +00:00
23c714da66 Add versioned analysis artifact state 2026-08-29 18:27:44 +00:00
6639775d7d Add the artifact regeneration command 2026-08-29 18:19:54 +00:00
966b95b176 Enforce bounded run prerequisites 2026-08-29 18:17:05 +00:00
3bcf2c08dd Add bounded run and plan commands 2026-08-29 18:07:24 +00:00
700ab655ca Add shared bounded pipeline plans 2026-08-29 18:02:37 +00:00
85c5647385 Separate stage order from invalidation dependencies 2026-08-29 18:00:22 +00:00
2ef7c76d99 Add post-transcript implementation plan 2026-08-29 17:47:15 +00:00
9bc1b0feda Plan post-transcript artifact regeneration 2026-08-29 17:29:08 +00:00
5cec84a4a7 Remove redundant prepared input APIs
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-08-29 16:19:22 +00:00
abfbe42d61 Snapshot verified references for extraction 2026-08-29 16:18:40 +00:00
e7e3bef1e4 Reject special files before preparing inputs 2026-08-29 16:13:36 +00:00
a68e8e31a4 Complete Notarius reference compatibility audit 2026-08-29 15:49:36 +00:00
3a9e60cda9 Document prepared Notarius references 2026-08-29 15:40:23 +00:00
905ff03ccc Prove prepared reference extraction lifecycle 2026-08-29 15:31:33 +00:00
495f7bcde4 Bind prepared references to extraction identity 2026-08-29 15:24:31 +00:00
51e0e8c5d0 Pass reference bindings to Notarius 2026-08-29 15:16:06 +00:00
a2409a1fd1 Verify prepared inputs from manifest evidence 2026-08-29 15:11:59 +00:00
b3363f87d6 Prepare optional spell catalog inputs 2026-08-29 15:03:44 +00:00
5831c0c9e6 Add Notarius reference configuration vocabulary 2026-08-29 14:55:24 +00:00
42ed81cbe1 Update Notarius integration and plan references 2026-08-29 14:41:22 +00:00
e433c86203 Close out the completed audit 2026-08-11 12:53:31 +00:00
a2a144dffa Close diagnostic and restore coverage gaps 2026-08-11 12:28:55 +00:00
8ef6e99d69 Bound remote control object reads 2026-08-11 03:43:18 +00:00
2545faef6c Dispose subprocess descendants after leader exit 2026-08-11 03:22:21 +00:00
80be8be4d6 Confine subprocess diagnostics and retain redacted tails 2026-08-11 03:11:27 +00:00
801adb385d Reconcile lifecycle documentation 2026-08-10 23:18:29 +00:00
feba7b9d74 Enforce continuous validation 2026-08-10 23:08:52 +00:00
b89224bbde Simplify stage and Audita contracts 2026-08-10 22:43:22 +00:00
131ffd9887 Make analyze input resolution deterministic 2026-08-10 22:36:21 +00:00
af492c9e97 Centralize effective artifact selection 2026-08-10 22:27:06 +00:00
f39fc94610 Bind extraction reuse to transcript identity 2026-08-10 22:15:53 +00:00
4e4eff6ba7 Centralize extraction bundle evidence 2026-08-10 22:06:48 +00:00
8ff1b4fa66 Enforce requested adapter output paths 2026-08-10 21:57:54 +00:00
702f622e18 Harden prepare and transcribe transitions 2026-08-10 21:53:54 +00:00
9da2c1e144 Stream WhisperX uploads safely 2026-08-10 21:49:44 +00:00
32653f54f9 Make configuration truthful and clean remote session files 2026-08-10 21:41:00 +00:00
72a200968a Harden configuration validation 2026-08-10 21:32:15 +00:00
b39b68add7 Unify previous source resolution 2026-08-10 21:20:59 +00:00
d9fa1d9328 Bind audio cache reuse to remote identity 2026-08-10 21:09:21 +00:00
8375ad83f3 Serialize restore recovery and rebase manifest paths 2026-08-10 21:01:28 +00:00
4158394dcf Bind restore to committed remote snapshots 2026-08-10 20:42:43 +00:00
eac7e155a5 Persist retryable post-publish cleanup obligations 2026-08-10 20:29:00 +00:00
0cf2cbfeb3 Make remote publish locks generation-safe 2026-08-10 20:17:54 +00:00
361dbb4ca8 Publish immutable remote commits 2026-08-10 20:05:24 +00:00
d6deccf3e8 Add immutable remote commit reader 2026-08-10 19:47:12 +00:00
ee747243fe Terminalize handled invocation failures 2026-08-10 19:34:46 +00:00
a1ceb457e9 Unify invocation manifest identity 2026-08-10 19:25:03 +00:00
9900211fa4 Confine publish archive reads 2026-08-10 19:16:18 +00:00
60cebf0e4b Redact and cap subprocess diagnostics 2026-08-10 18:55:49 +00:00
7bd575187e Terminate owned subprocess trees 2026-08-10 18:44:10 +00:00
ab5a7e8e3d Bound external result file reads 2026-08-10 18:30:28 +00:00
99b2e1cd81 Harden API key file loading 2026-08-10 18:24:29 +00:00
363313d99c Confine cleanup and use held session locks 2026-08-10 18:14:09 +00:00
18ddf00d3d Confine local file installation paths 2026-08-10 17:59:29 +00:00
59f3fe3d1d Make atomic file replacement crash durable 2026-08-10 17:44:38 +00:00
1dccf5f140 Validate portable workspace identifiers 2026-08-10 17:35:52 +00:00
0b40cf8026 Make ordinary workspaces group shareable 2026-08-10 17:27:00 +00:00
a7ec195587 Prepare a staged implementation plan to address the audit findings. 2026-08-10 17:08:57 +00:00
13de820931 Finalize codebase audit findings 2026-08-10 15:14:54 +00:00
14ef59aaed Document test suite policy audit conclusions 2026-08-10 15:04:05 +00:00
f387222fce Document maintainability audit conclusions 2026-08-10 14:53:19 +00:00
9cb9008dfc Document analyze dependency audit findings 2026-08-10 14:38:49 +00:00
083decc5b4 Document extraction audit findings 2026-08-10 14:24:03 +00:00
57cac5d3f7 Document ordinary stage audit findings 2026-08-10 14:09:54 +00:00
0a772e03b4 Document external adapter audit findings 2026-08-10 13:53:49 +00:00
0920062a38 Document configuration and composition audit findings 2026-08-10 13:33:59 +00:00
39afe644eb Document restore and previous-state audit findings 2026-08-10 13:16:35 +00:00
e3ee3de10a Audit publish commit and cleanup behavior 2026-08-10 12:55:10 +00:00
9c72db56e9 Audit artifact paths and filesystem safety 2026-08-10 12:42:49 +00:00
bb2d606dbb Audit runner and manifest lifecycle behavior 2026-08-10 12:28:36 +00:00
9850767a8a Establish the codebase audit baseline and contract map 2026-08-10 12:12:48 +00:00
74e2d21de5 Close the completed roadmap documents 2026-08-10 03:24:49 +00:00
7cb18a1a40 Reconcile promotion and manifest documentation 2026-08-10 03:04:20 +00:00
b556fc2f4f Clear superseded session stage result details 2026-08-10 02:53:28 +00:00
b99bd38eb4 Harden bundle promotion against symlink replacement 2026-08-10 02:45:51 +00:00
701b6726d7 Reconcile Notarius extraction documentation 2026-08-10 02:10:37 +00:00
665039f4dc Support atomic directory promotion across platforms 2026-08-10 01:58:52 +00:00
ef8dae776e Enforce canonical Notarius bundle paths 2026-08-10 01:50:31 +00:00
d01775b68a Exclude staged Notarius bundles from publish uploads 2026-08-10 01:43:43 +00:00
0d6f2dd0ce Invalidate downstream results when stages are replaced 2026-08-10 01:38:24 +00:00
df40cbec6e Document and validate Notarius extraction workflows 2026-08-10 00:42:44 +00:00
0341e0c7c0 Publish and inspect configured extraction artifacts 2026-08-10 00:32:24 +00:00
39af7d4f3c Integrate extraction artifacts into analysis catalog 2026-08-10 00:24:02 +00:00
bba582b4ca Integrate extraction lifecycle and resume validation 2026-08-10 00:14:46 +00:00
1f16a85330 Implement direct Notarius extraction execution 2026-08-09 23:59:50 +00:00
f9482639d4 Add the Notarius subprocess adapter 2026-08-09 23:47:38 +00:00
dce721cdbd Add safe immutable directory promotion 2026-08-09 23:37:02 +00:00
98734644d6 Add Notarius configuration and extraction source policy 2026-08-09 23:30:32 +00:00
951383226c Add artifact provenance and stage skip outcomes 2026-08-09 23:20:32 +00:00
df58595d1e Remove completed documentation alignment roadmaps 2026-08-09 22:02:07 +00:00
c3c14e7468 Complete documentation alignment roadmap 2026-08-09 21:53:04 +00:00
e7319ea016 Align internal documentation and maintained examples 2026-08-09 21:50:21 +00:00
bd2d5e2496 Clarify user and integration documentation contracts 2026-08-09 21:41:23 +00:00
115a44f629 Align documentation entry points and internal overview 2026-08-09 21:32:11 +00:00
18411dc5b5 Move contributor guidance to its canonical location 2026-08-09 21:27:58 +00:00
e23dc1ab6e Refocus the Narratio architecture policy 2026-08-09 21:26:18 +00:00
e1359ea227 Adopt canonical documentation ownership policy 2026-08-09 21:23:50 +00:00
7fdd99ec27 Prepare roadmap for documentation policy update 2026-08-09 21:21:00 +00:00
a90231ce0c Implement support for passing a session_id variable to scriptorium to support sticky routing 2026-07-02 21:04:37 -05:00
ed879b8bb0 Clean up obsolete placeholder code 2026-07-02 20:47:08 -05:00
717451512a Implemented new campaign/session stable inputs and corresponding input source references
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-27 09:34:05 -05:00
3ddb3a947b Update pipeline defaults so trim is enabled when omitted 2026-05-27 08:35:02 -05:00
427 changed files with 60196 additions and 11795 deletions

View File

@@ -2,39 +2,26 @@ when:
- event: tag - event: tag
steps: steps:
- name: build-release-assets validate-release:
image: golang:1.25 image: golang:1.26.7
commands:
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
build-release-assets:
image: golang:1.26.7
depends_on:
- validate-release
commands: commands:
- | - |
set -eu set -eu
case "$PWD" in
/*) ;;
*) echo "release workspace must have an absolute path" >&2; exit 1 ;;
esac
./scripts/build-release-assets.sh "$CI_COMMIT_TAG" "$PWD/dist"
version="$CI_COMMIT_TAG" publish-release:
dist="dist" image: woodpeckerci/plugin-release:0.3.1
pkg="gitea.maximumdirect.net/eric/narratio/cmd/narratio"
rm -rf "$dist"
mkdir -p "$dist"
build_binary() {
goos="$1"
goarch="$2"
suffix="$3"
output="$dist/narratio-$version-$goos-$goarch$suffix"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/narratio/internal/buildinfo.Version=$version" \
-o "$output" "$pkg"
}
build_binary linux amd64 ""
build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release
image: woodpeckerci/plugin-release
depends_on: depends_on:
- build-release-assets - build-release-assets
settings: settings:
@@ -42,6 +29,8 @@ steps:
from_secret: GITEA_RELEASE_TOKEN from_secret: GITEA_RELEASE_TOKEN
files: files:
- dist/narratio-* - dist/narratio-*
title: Narratio ${CI_COMMIT_TAG}
note: docs/releases/${CI_COMMIT_TAG}.md
checksum: sha256 checksum: sha256
checksum-file: SHA256SUMS checksum-file: SHA256SUMS
checksum-flatten: true checksum-flatten: true

8
.woodpecker/shuffle.yml Normal file
View File

@@ -0,0 +1,8 @@
when:
- event: cron
steps:
shuffled-race-tests:
image: golang:1.26
commands:
- go test -race -shuffle=on -count=3 ./...

47
.woodpecker/verify.yml Normal file
View File

@@ -0,0 +1,47 @@
when:
- event: [push, pull_request]
steps:
tests:
image: golang:1.26
commands:
- go test ./...
race-tests:
image: golang:1.26
depends_on: tests
commands:
- go test -race ./...
static-analysis:
image: golang:1.26
depends_on: tests
commands:
- go vet ./...
build:
image: golang:1.26
depends_on: tests
commands:
- go build ./...
documentation-and-examples:
image: golang:1.26
depends_on: tests
commands:
- go test ./internal/doccheck
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
cross-build:
image: golang:1.26
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
commands:
- |
set -eu
output_dir="$(mktemp -d)"
trap 'rm -rf "$output_dir"' EXIT
for target in linux/amd64 linux/arm64 darwin/arm64; do
goos="${target%/*}"
goarch="${target#*/}"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
done

View File

@@ -1,22 +1,42 @@
# narratio # narratio
Narratio is a stage-driven Go orchestrator for turning D&D session audio into polished transcripts and generated artifacts. Narratio is a stage-driven Go orchestrator for turning D&D session audio into
polished transcripts, validated Notarius extraction lanes, and generated
artifacts.
It runs a deterministic workflow across `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`, and `publish`, with manifest-driven continuation and restore support. It runs a deterministic workflow with manifest-driven continuation, remote
publish, and restore support.
```bash ```sh
narratio run 2026-04-04 narratio run 2026-04-04
``` ```
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete `session.yml` (or explicit config flags). This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete
`session.yml` files or their explicit command-line alternatives.
## Documentation ## Documentation
- [CLI Reference](docs/cli.md) - [CLI reference](docs/cli.md) — commands, arguments, flags, and invocation
- [Configuration](docs/config.md) behavior.
- [Operations](docs/operations.md) - [Configuration](docs/config.md) — discovery, fields, defaults, and
- [Troubleshooting](docs/troubleshooting.md) validation.
- [Internal Component Contracts](docs/internal/README.md) - [Operations](docs/operations.md) — runtime workflow, state, publishing,
- [Development Guide](docs/policy/development.md) recovery, and cleanup.
- [Architecture Principles](docs/policy/architecture.md) - [Troubleshooting](docs/troubleshooting.md) — symptom-driven diagnosis and
- [Maintained Examples](examples/) safe remedies.
- [Integration contracts](docs/integrations/) — external tools, formats, and
compatibility expectations.
- [Maintained examples](examples/README.md) — complete copyable configuration
and input files, including the production/testing split bundle.
## Maintainer Documentation
- [Development guide](docs/development.md) — first-read orientation and
task-specific reading routes.
- [Internal overview](docs/internal/overview.md) — implemented component map.
- [Architecture](docs/policy/architecture.md) — normative boundaries and
invariants.
- [Documentation policy](docs/policy/documentation.md) — canonical ownership
and maintenance rules.
- [Testing policy](docs/policy/testing.md) — test value, boundaries, and
sufficiency.

View File

@@ -12,12 +12,15 @@ This runs the canonical full pipeline for session `2026-04-04`.
Top-level commands: Top-level commands:
- `run <session_id>`: run full stage order. - `version`: print the Narratio build version.
- `run <session_id>`: run all or one contiguous range of the canonical stage order.
- `regenerate-artifacts <session_id>`: force-run extraction through analysis.
- `run-stage <stage> <session_id>`: run one stage. - `run-stage <stage> <session_id>`: run one stage.
- `analyze <session_id>`: force-run analyze. - `analyze <session_id>`: force-run analyze.
- `publish <session_id>`: force-run publish. - `publish <session_id>`: force-run publish.
- `clean <session_id>` or `clean --all`: remove local work/spool state. - `clean <session_id>` or `clean --all`: remove local work/spool state.
- `session <subcommand>`: session helper commands. - `session <subcommand>`: session helper commands.
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
Session subcommands: Session subcommands:
@@ -41,13 +44,26 @@ Most session-aware commands accept:
- `--session <session.yml>` - `--session <session.yml>`
- `--session-id <session_id>` - `--session-id <session_id>`
- `--previous-session-id <session_id>` - `--previous-session-id <session_id>`
- `--profile <name>`
- `--secrets-dir <directory>`
Rules: Rules:
- `--campaign` and `--campaign-file` are mutually exclusive. - `--campaign` and `--campaign-file` are mutually exclusive.
- `--session` is not used by `session init`. - `--session` is not used by `session init`.
- if both positional `<session_id>` and `--session-id` are provided, values must match. - if both positional `<session_id>` and `--session-id` are provided, values must match.
- `--previous-session-id` is a strict expectation: the selected session file
must contain the same `previous_session_id`.
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
may be supplied once; omitted, empty, and the exact name `default` select the
base configuration, while an unknown named profile fails. The same selection
applies to all common-flag commands, including `regenerate-artifacts`.
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
relative value uses caller CWD. Without the flag, a present `secrets/`
directory beside the resolved root pipeline is used; an absent one is skipped.
- `clean --all` cannot be combined with campaign/session selectors. - `clean --all` cannot be combined with campaign/session selectors.
- notification delivery is currently limited to the configured `noop` mode; see
the [configuration reference](./config.md#notifications).
## Session ID Input Rules ## Session ID Input Rules
@@ -66,18 +82,144 @@ Commands with additional positionals keep their command-specific order:
## Command Reference ## Command Reference
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
```bash
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
```
These commands resolve the selected profile, defaults, ordinary paths, and—if
a campaign is selected—the campaign-owned party. They neither discover or load
a session nor create a workspace, manifest, run, lock, adapter, or remote
connection. Like every pipeline-loading command, they bootstrap the selected or
conventional secrets directory before resolving configuration; `config show`
never includes bootstrap reports or secret values in its YAML output.
Typed credential references, including `notarius.llm_api_key`, must therefore
resolve even though these inspection commands never construct an adapter.
Campaign selection is optional for a pipeline without party-driven artifact
families. A pipeline with `scriptorium.artifact_families` needs a selected or
configured default campaign so Narratio can expand its concrete artifacts and
publish rules. `--campaign` and `--campaign-file` remain mutually exclusive.
Session, range, force, and artifact-execution flags are not accepted.
`config validate` writes a concise root-path, selected-profile (or `none`), and
effective-digest summary after successful complete validation. `config show`
writes one deterministic, secret-free YAML document containing defaulted and
expanded concrete configuration. It omits ConfigKit directives, artifact
family declarations, and runtime provenance.
`config sources` reports the same fully validated resolution without printing
effective values. Its header identifies the root, ordered imports, selected
profile and overlay, selected campaign, party source, and digest. The
remaining tab-separated records are sorted as `path`, `role`, and `source`.
Roles distinguish root, import, profile, centralized default, campaign, party,
and generated family ownership. A generated party member has
one family record and one party record at the same logical path. The output
never reads or prints secret values.
`config artifacts` prints the concrete catalog used by analysis, ordered by
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
`party_source`. Definitions and effective dependencies are sorted comma-separated
lists, and control characters in fields are escaped. Family-member rows identify
their family, character, defining family file, and party file. An empty catalog
prints only the header. Ordinary artifacts can be inspected without a campaign;
party-driven families retain the campaign requirement. See
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
configuration layout.
`config diff` resolves both supplied profile names independently and compares
their fully resolved, secret-free effective mappings. It does
not accept `--profile`; the two positional names must be distinct, declared
profiles. When party-driven families are present, both profiles must resolve to
the same selected campaign and party. Use `--campaign-file` if profile-specific
campaign configuration would otherwise select different files.
Equal profiles print `no differences`. Otherwise, sorted tab-separated records
use one of these forms, with compact JSON values:
```text
added <path> <right-value>
removed <path> <left-value>
changed <path> <left-value> <right-value>
```
Mappings are flattened to their logical field paths; lists remain one atomic
value. The command compares defaulted concrete artifacts and publish rules, not
profile names, source-file layout, or formatting. It succeeds when differences
are found, making it suitable for review and migration checks.
### `version`
```bash
narratio version
```
Official release binaries report their exact Git tag. Binaries built directly
from source without release linker metadata report `dev`.
### `run` ### `run`
```bash ```bash
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags] narratio run <session_id> [--from <stage>] [--through <stage>] [--force] [--artifacts <name[,name...]>] [...common config flags]
``` ```
Behavior: Behavior:
- evaluates full stage order; - evaluates one inclusive contiguous range of the canonical stage order;
- skips already-succeeded stages unless `--force` is set; - defaults an omitted `--from` to `prepare` and an omitted `--through` to
`cleanup`, so omitting both retains full-pipeline behavior;
- rejects unknown endpoints and a `--from` endpoint after `--through`;
- runs `render` before `extract`; an omitted or disabled Notarius
configuration records an explicit `notarius_disabled` self-skip;
- resolves every selected typed credential during the initial configuration
load, including a Notarius credential when extract is disabled, outside the
selected range, or later skipped as resumable;
- skips already-succeeded stages unless `--force` is set or a stage-specific
resume check finds its durable result obsolete;
- applies `--force` only to stages in the selected range;
- rejects repeated `--from`, `--through`, or `--force` options, including
`--name=value` spellings;
- continues interrupted or partially completed sessions by running non-succeeded stages; - continues interrupted or partially completed sessions by running non-succeeded stages;
- writes session and run manifests. - writes the session manifest as the sole durable progress ledger. Each
invocation also creates its run-local directory and returns an immediate
command summary; neither creates another durable progress record.
- reports the resolved profile (or `none`) and effective configuration digest.
When `--artifacts` is present, the selected range must contain `analyze` or
`publish`. Either consumer is sufficient, including a one-stage range.
### `regenerate-artifacts`
```bash
narratio regenerate-artifacts <session_id> [--artifacts <name[,name...]>] [...common config flags]
```
Exactly equivalent to:
```bash
narratio run <session_id> --force --from extract --through analyze [caller options]
```
The command always reruns extraction. Analysis rebuilds the selected configured
artifacts and any prerequisites required by those targets; without
`--artifacts`, it uses the normal default analysis selection. Publish and notify
never run. Common session/configuration options and repeatable artifact values
pass through unchanged.
The forced extraction rebuilds the complete configured Notarius bundle,
disables Notarius checkpoint loading, and requests chunk-plan refresh.
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
supply those options. The shared `run` parser reports them as duplicate
singleton flags. The alias has no private execution options or behavior, and
runtime diagnostics may identify the operation as `run`.
### `run-stage` ### `run-stage`
@@ -94,9 +236,11 @@ Valid stage names:
- `normalize` - `normalize`
- `trim` - `trim`
- `render` - `render`
- `extract`
- `analyze` - `analyze`
- `publish` - `publish`
- `notify` - `notify`
- `cleanup`
Rules: Rules:
@@ -114,6 +258,10 @@ Equivalent to:
narratio run-stage analyze <session_id> --force [...common config flags] narratio run-stage analyze <session_id> --force [...common config flags]
``` ```
This selects only `analyze`: it does not invoke Notarius, replace extraction,
or remove existing extract outputs. `--artifacts` narrows the forced analysis
targets and their required analysis prerequisites.
### `publish` ### `publish`
```bash ```bash
@@ -126,6 +274,10 @@ Equivalent to:
narratio run-stage publish <session_id> --force [...common config flags] narratio run-stage publish <session_id> --force [...common config flags]
``` ```
This command selects only `publish`; it does not invoke cleanup. Use a normal
full run to include configured cleanup, or `run-stage cleanup` to retry cleanup
from valid publication evidence.
### `clean` ### `clean`
```bash ```bash
@@ -135,21 +287,29 @@ narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
Behavior: Behavior:
- session mode removes: - session mode removes the selected session's local work and spool state;
- `{workspace.root}/work/{campaign}/{session_id}` - `--all` removes all local session work and spool state;
- `{spool.root}/{campaign}/{session_id}`
- `--all` removes:
- `{workspace.root}/work/*`
- direct children under `{spool.root}`
- cache remains unless `--clear-cache` is provided. - cache remains unless `--clear-cache` is provided.
See [Operations: Cleanup](./operations.md#cleanup) for manual and pipeline-stage
deletion scope.
### `session plan` ### `session plan`
```bash ```bash
narratio session plan <session_id> [--force] [...common config flags] narratio session plan <session_id> [--from <stage>] [--through <stage>] [--force] [--artifacts <name[,name...]>] [...common config flags]
``` ```
Validates config, prepares local workdir layout, and prints run/skip decisions for each stage. Uses the same inclusive bounds, endpoint validation, force scope, and artifact
selection contract as `run`. It validates config and prints run/skip decisions
for selected stages only without creating the local workdir or changing the
manifest. Resume-capable selected stages are checked against durable evidence.
The output includes the resolved profile (or `none`) and effective configuration
digest without writing provenance or any manifest state.
For `analyze`, the preview also lists explicit targets, prerequisite-only work,
execution order, and reusable current artifacts with concise reasons. These
artifact decisions come from the same reconciliation and work planner used by
execution; the preview does not predict output identities.
### `session validate` ### `session validate`
@@ -157,7 +317,8 @@ Validates config, prepares local workdir layout, and prints run/skip decisions f
narratio session validate <session_id> [...common config flags] narratio session validate <session_id> [...common config flags]
``` ```
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks. Read-only preflight checks for config validity, required inputs, audio mode,
previous-session requirements, publish outputs, and independent remote locks.
### `session status` ### `session status`
@@ -165,7 +326,13 @@ Read-only preflight checks for config validity, required inputs, audio mode, pre
narratio session status <session_id> [...common config flags] narratio session status <session_id> [...common config flags]
``` ```
Prints local manifest state and, when storage is available, remote current-state and published-output status. Prints local manifest state and, when storage is available, status for the
pointer-selected remote commit and its declared published outputs. Current
publish metadata reports `published`, `carried_locked`, `locked_unpublished`,
`optional_missing`, and `unselected`; carried output lines include the run
derived from their exact declared key, the key itself, and its declared
destination. A separately displayed `dest` is the destination requested by the
current configuration and may differ while an older declaration is locked.
### `session init` ### `session init`
@@ -184,6 +351,7 @@ Options:
- `--config <pipeline.yml>` - `--config <pipeline.yml>`
- `--campaign <id>` or `--campaign-file <campaign.yml>` - `--campaign <id>` or `--campaign-file <campaign.yml>`
- `--secrets-dir <directory>`
- `--previous-session-id <id>` - `--previous-session-id <id>`
- `--date <YYYY-MM-DD>` - `--date <YYYY-MM-DD>`
- `--title <text>` - `--title <text>`
@@ -207,17 +375,12 @@ Behavior:
- discovers committed remote current state; - discovers committed remote current state;
- plans local restores; - plans local restores;
- writes `reports/restore-latest.json` on execution; - writes an execution report;
- blocks conflicting overwrites unless `--force` is set. - blocks unresolved conflicts. `--force` permits replacement only of eligible
regular files.
Default restore scope: See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
default restore scope, report location, and conflict-handling workflow.
- `manifest.json`
- `transcripts/**`
- `artifacts/**`
- `previous/**` when required by configured previous-session inputs
`audio/**` is included only with `--include-audio`.
### `session artifacts` ### `session artifacts`
@@ -225,7 +388,11 @@ Default restore scope:
narratio session artifacts <session_id> [--remote] [...common config flags] narratio session artifacts <session_id> [--remote] [...common config flags]
``` ```
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability. Lists effective built-in, configured Scriptorium, and configured extraction
sources; reports planned, available, unavailable, and published state without
reading payload bodies; and includes publish rules, lock state, and optional
remote published-state availability. Remote inspection uses exact commit
declarations and displays the five current publish outcomes when recorded.
### `session locks` ### `session locks`
@@ -237,22 +404,38 @@ narratio session locks remove <session_id> <source> [...common config flags]
Behavior: Behavior:
- list mode merges static `pipeline.publish.locks` with remote `{session_prefix}/locks.yml`; - list mode validates and reports independent remote locks sorted by source;
- add/remove mutate only remote locks; - add creates one lock and reports an existing lock unless `--force` is used;
- static locks from pipeline config cannot be removed by CLI commands. - `--force` performs one generation-matched replacement and reports a
concurrent change for operator retry; and
- remove is idempotent and reports when the source was not locked.
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
and publication behavior.
## `--artifacts` Selection Rules ## `--artifacts` Selection Rules
- accepted on `run`, `run-stage`, `analyze`, and `publish`; An artifact-family key selects all of its concrete character members. A
concrete generated key selects only that member; mixed family and concrete
selection is deduplicated and executed as concrete keys. The resulting plan
and command output identify both the concrete key and, where applicable, its
family and character ID.
- accepted on `run`, `session plan`, `run-stage`, `analyze`, and `publish`;
- repeatable and comma-separated values are combined, surrounding whitespace
is removed, and duplicate names are collapsed;
- names must exist in `pipeline.scriptorium.artifacts`; - names must exist in `pipeline.scriptorium.artifacts`;
- empty entries are invalid; - empty entries are invalid;
- repeated names are deduplicated. - on `run-stage`, only `analyze` and `publish` accept the option.
Effects: Effects:
- filters analyze execution to selected configured artifacts; - selects explicit analyze targets; required configured prerequisites may be
reused or rebuilt before them;
- filters publish rules that source `narratio.artifact.<name>`; - filters publish rules that source `narratio.artifact.<name>`;
- does not filter built-in transcript/bounds publish sources. - does not filter built-in transcript/bounds or explicitly configured
`narratio.extraction.<name>` publish sources; and
- does not select or filter Notarius lanes.
## Common Workflows ## Common Workflows
@@ -279,3 +462,24 @@ Force publish only:
```bash ```bash
narratio publish 2026-04-04 narratio publish 2026-04-04
``` ```
Regenerate post-transcript artifacts without publishing:
```bash
narratio regenerate-artifacts 2026-04-04 --artifacts session_recap,player_handout
```
## Output And Exit Behavior
- Successful commands write their result or summary to standard output and
exit with status `0`.
- Command failures and invalid invocations write an error to standard error and
exit with status `1`.
- An unknown top-level command also prints the top-level usage summary to
standard error.
- `session restore --help` prints its command-specific usage and exits with
status `0`.
Output is intended for operator inspection. Narratio does not currently offer
a machine-readable CLI output mode; durable machine-readable state is recorded
in manifests and reports described in [Operations](./operations.md).

View File

@@ -38,17 +38,159 @@ If local session discovery fails and a `session_id` is known, Narratio attempts
using configured object storage. using configured object storage.
The downloaded remote session file is command-scoped: Narratio removes it after
the command finishes and records only the remote object provenance alongside
the durable copied session input.
### Read-only effective pipeline inspection
`narratio config validate`, `narratio config show`, `narratio config sources`,
and `narratio config artifacts` use the same `--config`, `--campaign`,
`--campaign-file`, and
`--profile` selection rules as pipeline commands, but do not select, discover,
or load a session. They resolve configured typed credential references as part
of the ordinary ConfigKit load but expose only their environment-variable names;
they do not create runtime state.
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
campaign selectors, resolves each named profile with an independent ConfigKit
load, and does not accept a separate `--profile` flag.
Campaign selection is optional only when the resolved pipeline has no
`scriptorium.artifact_families`. When families are declared, Narratio selects a
campaign through an explicit flag or `pipeline.campaigns.default_campaign_id`,
then parses the campaign-owned party and expands concrete artifacts and any
family publish rules before validation. `config validate` prints the resulting
root, profile, and effective digest. `config show` emits the normalized
effective pipeline YAML, with defaults and concrete expansion included but
ConfigKit directives and family declarations omitted. This review projection is
not promised to be a reloadable authoring document. `config sources` prints a stable
source projection instead of effective values: root/import/profile/default
ownership plus campaign/party and generated-family records. Canonical derived
players trace to the party. `config artifacts` lists the concrete artifact map,
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
owns command syntax and output conventions.
`config diff` compares normalized field values rather than YAML text or source
ownership. It emits sorted `added`, `removed`, and `changed` records, uses
compact deterministic JSON values, treats lists atomically, and reports `no
differences` when the complete effective configurations are equal. Concrete
family members and generated publish rules participate after expansion; moving
an equal value between eligible root/import sources does not create a
difference.
### Migrating to the maintained bundle
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
as the complete copyable migration reference. Keep production settings in the
base, split them into explicit additive imports, and place testing differences
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
in the recommended one-definition files shown by that bundle. Convert campaign
rosters to [canonical party input](integrations/party.md), then express character
work as families. Inspect the result with `config validate`, `config show`,
`config sources`, and `config artifacts`; use `config diff default testing` to
review the overlay before running a session. Unversioned party documents are
rejected.
### Identity segments
Campaign IDs (`campaign_id` and `default_campaign_id`), session IDs, previous
session IDs, and Narratio run IDs are opaque portable segments. They must use
only ASCII letters, digits, `.`, `_`, and `-`; empty values, `.`/`..`, path
separators, drive forms, whitespace, control characters, and non-ASCII text are
rejected. Narratio does not trim or rewrite these values. Existing manifests or
remote state with an unsafe legacy identity must be migrated before use.
## Validation and Merge Rules ## Validation and Merge Rules
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load. - ConfigKit performs strict typed YAML decoding and accepts exactly one document;
unknown fields, duplicate keys, and trailing documents fail load.
- A pipeline file may explicitly import additive YAML fragments through the
root-only `configkit.imports` list. Imported files contribute fields to one
logical pipeline document; they do not override fields supplied by the root
or another import.
- An omitted profile, an empty profile option, and the exact name `default`
select the base configuration. Any other name selects
`profiles/<name>.yml` relative to the pipeline root.
- Campaign, session, and canonical-party documents are strict single-source
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
directives.
- Configured timeout and retry-delay durations must be positive. An omitted
artifact timeout continues to inherit its configured Scriptorium timeout.
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load. - Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
- Session and party inputs inspected before typed loading are limited to 8 MiB.
- Pipeline defaults are applied before validation. - Pipeline defaults are applied before validation.
- Defaults apply only to absent fields. Explicit empty strings, zero values,
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
Narratio domain validation; they are not silently replaced by defaults.
Explicit null optional stage sections stay absent, while required sections
fail validation.
- Campaign and session identities must agree. - Campaign and session identities must agree.
- Stable files (`speakers_file`, `autocorrect_file`, `glossary_file`) resolve from session overrides when provided, otherwise from campaign defaults. - Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
`party_file`) are campaign-owned; the first three may be overridden by the
session. The optional `spell_catalog_file` also inherits a campaign value
when omitted by the session.
- `party_file` is parsed when pipeline and campaign configuration are combined.
It must be a versioned [canonical party](integrations/party.md), is
campaign-owned and derives the players input internally. `players_file` and
session `party_file` are unsupported configuration keys.
- Exactly one audio mode must be configured in session input: - Exactly one audio mode must be configured in session input:
- local (`audio_dir` or `audio_files`), or - local (`audio_dir` or `audio_files`), or
- S3 (`audio_s3.prefix`). - S3 (`audio_s3.prefix`).
### Pipeline imports and profiles
Large pipeline configurations may be split into explicitly named fragments and
may use fixed-path profile overlays:
```yaml
configkit:
imports:
- path: config/storage.yml
- path: config/integrations.yaml
campaigns:
root: /usr/local/share/narratio/campaigns
```
The maintained [production/testing bundle](../examples/production-testing/pipeline.yml)
is a complete copyable example of this structure, including canonical-party
artifact families.
Imports are resolved relative to the directory containing the root pipeline
file and are loaded in declaration order. Narratio does not scan directories or
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
absolute paths, traversal, symlinks, directories, duplicate files, and an
import of the root pipeline itself are rejected. Imported files cannot declare
another `configkit` control section.
Base merging is additive. A map may be extended by multiple files when every
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
more than once, even when the repeated values are identical. Conflict errors
name the full field path and every source that claimed it. Narratio additionally
rejects an artifact or artifact-family key declared by more than one base file,
even when those files contribute disjoint children.
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
rules. Only the selected overlay is opened; a malformed unselected overlay does
not affect a base or different-profile load. Maps merge recursively, overlay
scalars replace base scalars, and overlay lists replace base lists completely.
Explicit `false`, zero, empty strings, empty containers, and null values remain
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
stack, import files, or declare another ConfigKit control section.
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
entry-level defaults and normalization, read-only file-local validation, and
ordinary path resolution before computing a deterministic effective
configuration digest. The digest represents the normalized, secret-free
runtime pipeline mapping; it excludes ConfigKit directives, source
provenance, profile identity, and raw environment secret values. Equivalent
effective mappings therefore have the same digest regardless of how fields are
split among the root and imports.
An imported field has the same meaning it would have in a monolithic root
pipeline. In particular, ordinary relative pipeline paths continue to resolve
from the root pipeline directory, not from the importing fragment's directory.
## Minimal Working Configuration ## Minimal Working Configuration
`pipeline.yml` `pipeline.yml`
@@ -69,6 +211,7 @@ inputs:
speakers_file: ./speakers.yml speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml glossary_file: ./glossary.yml
party_file: ./party.yml
``` ```
`session.yml` (local audio) `session.yml` (local audio)
@@ -82,9 +225,31 @@ inputs:
## Secrets Handling ## Secrets Handling
- Do not place raw secrets in YAML. - Do not place raw secrets in YAML.
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`). - Credential fields contain environment-variable names, represented internally
- Optionally load env files from `pipeline.secrets.env_dir`. as ConfigKit typed secret references (for example
- Commands that need storage/auth load filesystem secrets before constructing adapters. `pipeline.notarius.llm_api_key: NOTARIUS_LLM_API_KEY`). A configured reference
must resolve to a nonempty environment value during load.
- Every surviving reference is resolved during ordinary pipeline loading,
regardless of whether its integration is disabled, outside the selected
stage range, or eligible for resume. Omit the field or set it to `null` when
that deployment should not require the credential.
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
flag is omitted, Narratio uses a present `secrets/` directory beside the
resolved root `pipeline.yml`; an absent conventional directory is ignored.
Explicit relative paths are resolved from the caller's working directory,
and a missing or empty explicit selection is an error.
- Each immediate regular file whose name is a valid environment-variable name
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
selected directory, rejects non-hidden symlink entries, and limits input to
1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
files, directories, and symlinks) are ignored and omitted from bootstrap
reports, but still count toward the directory entry limit.
- An existing process environment value takes precedence over a credential file.
- On POSIX, operators must provision the directory with no group/other access
(normally `0700`) and files with no group/other access (normally `0600`);
ConfigKit validates type and bounds but does not enforce permission modes.
- Bootstrap happens once per command, before any configuration load or adapter
construction. Safe reports contain filenames, paths, and statuses, never values.
## Publish Configuration Summary ## Publish Configuration Summary
@@ -107,17 +272,18 @@ publish:
- source: narratio.artifact.session_recap - source: narratio.artifact.session_recap
dest: artifacts/session_recap.md dest: artifacts/session_recap.md
required: true required: true
locks:
- source: narratio.artifact.session_recap
reason: manual post-publish edits
``` ```
Rules: Rules:
- `outputs[].source` is required. - `outputs[].source` is required.
- `outputs[].dest` may be omitted when derivable from source. - `outputs[].dest` may be omitted when derivable from source.
- extraction sources require an explicit `outputs[].dest` and publish only when
a rule names that source; the Notarius index and complete bundle are not
publish sources.
- `outputs[].required` defaults to `true`. - `outputs[].required` defaults to `true`.
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates. - publish locks are independent remote session objects managed with
`narratio session locks`; they are not pipeline configuration fields.
## Full Schema ## Full Schema
@@ -125,21 +291,21 @@ Rules:
| Field | Type | Required | Default / Rule | | Field | Type | Required | Default / Rule |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` | | `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` | | `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` | | `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
| `pipeline.campaigns.default_campaign_id` | string | No | empty | | `pipeline.campaigns.default_campaign_id` | string | No | empty |
| `pipeline.secrets.env_dir` | string | No | empty | | `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
| `pipeline.storage.backend` | string | No | empty | | `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` | | `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
| `pipeline.storage.s3.region` | string | No | empty | | `pipeline.storage.s3.region` | string | No | empty |
| `pipeline.storage.s3.endpoint` | string | No | empty | | `pipeline.storage.s3.endpoint` | string | No | empty |
| `pipeline.storage.s3.force_path_style` | bool | No | `false` | | `pipeline.storage.s3.force_path_style` | bool | No | `false` |
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` | | `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` | | `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
| `pipeline.spool.root` | string | No | `/var/spool/narratio` | | `pipeline.spool.root` | string | No | `/var/spool/narratio` |
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` | | `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.cache.root` | string | No | `/var/cache/narratio` | | `pipeline.cache.root` | string | No | `/var/cache/narratio` |
| `pipeline.cache.s3_audio` | bool | No | `true` | | `pipeline.cache.s3_audio` | bool | No | `true` |
| `pipeline.publish.enabled` | bool | No | `true` | | `pipeline.publish.enabled` | bool | No | `true` |
@@ -148,10 +314,7 @@ Rules:
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source | | `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation | | `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
| `pipeline.publish.outputs[].required` | bool | No | `true` | | `pipeline.publish.outputs[].required` | bool | No | `true` |
| `pipeline.publish.locks[]` | list | No | empty | | `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
| `pipeline.publish.locks[].reason` | string | No | empty |
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
| `pipeline.whisperx.language` | string | No | `en` | | `pipeline.whisperx.language` | string | No | `en` |
| `pipeline.whisperx.timeout` | duration | No | `30m` | | `pipeline.whisperx.timeout` | duration | No | `30m` |
| `pipeline.whisperx.retries` | int | No | `3` | | `pipeline.whisperx.retries` | int | No | `3` |
@@ -168,7 +331,7 @@ Rules:
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset | | `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
| `pipeline.audita.binary` | string | No | `audita` | | `pipeline.audita.binary` | string | No | `audita` |
| `pipeline.audita.timeout` | duration | No | `3h` | | `pipeline.audita.timeout` | duration | No | `3h` |
| `pipeline.audita.llm_api_key_env` | string | No | empty | | `pipeline.audita.llm_api_key` | secret reference | No | unset |
| `pipeline.audita.modules[]` | list[string] | No | empty | | `pipeline.audita.modules[]` | list[string] | No | empty |
| `pipeline.audita.base_url` | string | No | empty | | `pipeline.audita.base_url` | string | No | empty |
| `pipeline.audita.model` | string | No | empty | | `pipeline.audita.model` | string | No | empty |
@@ -184,16 +347,28 @@ Rules:
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` | | `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` | | `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
| `pipeline.normalize.report` | bool | No | `true` | | `pipeline.normalize.report` | bool | No | `true` |
| `pipeline.trim.enabled` | bool | No | `false` | | `pipeline.trim.enabled` | bool | No | `true` |
| `pipeline.trim.output_path` | string | Conditional | required when trim enabled | | `pipeline.trim.output_path` | string | No | `transcripts/final.trimmed.json` |
| `pipeline.trim.bounds.prompt_id` | string | Conditional | required when trim enabled | | `pipeline.trim.bounds.prompt_id` | string | No | `dnd.session_bounds` |
| `pipeline.trim.bounds.profile_id` | string | No | empty | | `pipeline.trim.bounds.profile_id` | string | No | empty |
| `pipeline.trim.bounds.transcript_input_name` | string | Conditional | required when trim enabled | | `pipeline.trim.bounds.transcript_input_name` | string | No | `transcript` |
| `pipeline.trim.bounds.output_path` | string | Conditional | required when trim enabled | | `pipeline.trim.bounds.output_path` | string | No | `artifacts/session_bounds.json` |
| `pipeline.trim.bounds.timeout` | duration | No | `10m` | | `pipeline.trim.bounds.timeout` | duration | No | `10m` |
| `pipeline.trim.bounds.render_debug` | bool | No | `false` | | `pipeline.trim.bounds.render_debug` | bool | No | `false` |
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true | | `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
| `pipeline.trim.seriatim.report` | bool | No | `false` | | `pipeline.trim.seriatim.report` | bool | No | `false` |
| `pipeline.notarius.enabled` | bool | No | `false` |
| `pipeline.notarius.binary` | string | No | `notarius` |
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
| `pipeline.render.enabled` | bool | No | `true` | | `pipeline.render.enabled` | bool | No | `true` |
| `pipeline.render.format` | string | No | `markdown` (only supported value) | | `pipeline.render.format` | string | No | `markdown` (only supported value) |
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) | | `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
@@ -205,9 +380,133 @@ Rules:
| `pipeline.scriptorium.timeout` | duration | No | `10m` | | `pipeline.scriptorium.timeout` | duration | No | `10m` |
| `pipeline.scriptorium.render_debug` | bool | No | `false` | | `pipeline.scriptorium.render_debug` | bool | No | `false` |
| `pipeline.scriptorium.artifacts` | map | No | empty | | `pipeline.scriptorium.artifacts` | map | No | empty |
| `pipeline.notification.backend` | string | No | empty | | `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
| `pipeline.notification.recipient` | string | No | empty | | `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
| `pipeline.notification.timeout` | duration | No | `30s` |
### Notarius Configuration Profile
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
configuration profile when extraction executes:
```yaml
notarius:
config_profile: downstream-economy
```
When the field is omitted or `null`, Narratio supplies no downstream profile
selector and preserves Notarius's existing configuration-selection behavior.
When present, Narratio forwards the exact string to Notarius, including an
empty string or `default`, which explicitly select Notarius's composed base
configuration. Notarius owns selector interpretation and validation; Narratio
does not restrict profile names, resolve profile files, or rewrite the value.
This selection is independent of Narratio's own `--profile` selection. A
Narratio overlay may explicitly select a differently named Notarius overlay,
but Narratio never forwards its own profile name automatically. It is also
independent of `pipeline.notarius.llm_api_key`: an overlay may change either or
both fields according to the deployed Notarius configuration. See the
[Notarius integration contract](./integrations/notarius.md) for invocation and
failure behavior.
### Notarius Credential
`pipeline.notarius.llm_api_key` optionally names the one environment variable
to forward to the Notarius child process:
```yaml
notarius:
llm_api_key: OPENROUTER_API_KEY
```
The field contains the variable name, not a credential value. That name must
match `api_key_env` in the Notarius/PromptKit profile selected by the deployed
Notarius pipeline; Narratio does not infer or remap provider names. Omission or
`null` forwards nothing. A non-null selected reference must resolve at pipeline
load time through the process environment or existing secrets-directory
bootstrap, even when Notarius is disabled, extract is outside the requested
range, or a completed extraction can be reused.
The Notarius child receives only Narratio's minimal execution environment plus
this exact selected name and loaded value. Other credentials are not forwarded.
Changing the value, name, presence, or absence does not invalidate extract's
semantic resume evidence. The public name remains part of safe effective
inspection, so renaming it may appear in `config diff` and change the effective
configuration digest; rotating only its value does neither.
See the [Notarius integration contract](./integrations/notarius.md) for the
delivery and redaction boundary.
### Notarius Cache Controls
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
extraction passes `--resume`; the deployed Notarius configuration must already
enable checkpoint recording and should select an explicit persistent checkpoint
root. Narratio does not inspect or generate that configuration.
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
configured mode in control. A nonempty value must be `auto`, `bypass`, or
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
Neither field changes Narratio's semantic extraction fingerprint, so changing
them alone does not rerun an otherwise complete durable extraction.
### Notarius Reference Bindings
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
Narratio source, not to a filesystem path:
```yaml
notarius:
references:
glossary: narratio.input.glossary
party: narratio.input.party
players: narratio.input.players
spell_catalog: narratio.input.spell_catalog
```
Supported sources are `narratio.input.party`, `narratio.input.players`,
`narratio.input.glossary`, and `narratio.input.spell_catalog`. Each map entry is
required by its presence: omit a binding when the selected Notarius pipeline
does not need it. A spell-catalog binding additionally requires an effective
campaign or session `spell_catalog_file`.
Selectors accept Notarius's `slot`, `chunk.slot`, `lane.slot`,
`lane.extract.slot`, `lane.merge.slot`, and `lane.normalize.slot` forms.
Narratio trims whitespace around
selectors and their dot-separated components, rejects empty components and
`=`, rejects duplicate normalized selectors, and limits the map to 256 entries.
It validates only selector structure and the prepared source vocabulary;
Notarius owns target-slot declarations and media compatibility.
Before extraction, Narratio resolves every binding from the current prepared
session manifest and streams it into a verified, content-addressed session
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
changed-during-copy, or checksum-inconsistent prepared evidence fails with
guidance to force `prepare`. Bindings are sorted by normalized selector and are
part of extraction fingerprint and resume identity. See the
[Notarius integration contract](./integrations/notarius.md) for the subprocess
boundary and the [complete example](../examples/pipeline.full.annotated.yml)
for a copyable configuration.
### Notarius Output Entries
For each `pipeline.notarius.outputs.<name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `lane_id` | string | Yes | unique Notarius lane ID |
| `media_type` | string | Yes | exact accepted descriptor media type |
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
| `schema_version` | string | Yes | exact accepted descriptor schema version |
| `module_key` | string | No | exact accepted module key when set |
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
required from a successful Notarius result; a missing, rejected, duplicate, or
contract-incompatible lane fails extraction. See the
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
for compatibility ownership.
### Scriptorium Artifact Entries ### Scriptorium Artifact Entries
@@ -216,46 +515,118 @@ For each `pipeline.scriptorium.artifacts.<name>`:
| Field | Type | Required | Rule | | Field | Type | Required | Rule |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `enabled` | bool | No | `false` if omitted | | `enabled` | bool | No | `false` if omitted |
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; enabled graph must be acyclic | | `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
| `render_debug` | bool | No | per-artifact override | | `render_debug` | bool | No | per-artifact override |
| `prompt_id` | string | Conditional | required when artifact is enabled | | `prompt_id` | string | Conditional | required when artifact is enabled |
| `profile_id` | string | No | empty | | `profile_id` | string | No | empty |
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules | | `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
| `timeout` | duration | No | artifact override | | `timeout` | duration | No | artifact override |
| `inputs` | map | No | input key names must be non-empty | | `inputs` | map | No | input key names must be non-empty |
| `vars` | map | No | values must be string or bool | | `vars` | map | No | values must be string or bool; `session_id` is reserved and overwritten by Narratio |
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
Without `--artifacts`, analyze executes enabled configured artifacts. With an
explicit `--artifacts` list, the exact named configured artifacts are the
one-invocation targets even if their `enabled` values are false. Analyze closes
those targets over the effective dependency graph: a current prerequisite is reused, while a
stale, missing, or failed prerequisite is rebuilt before its dependent.
Unrelated artifacts are not executed. Named targets and any prerequisite that
may require rebuilding must therefore have valid executable fields. This
override affects analyze planning only; publish uses the list only to filter
configured `narratio.artifact.<name>` output rules.
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`: For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
| Field | Type | Required | Rule | | Field | Type | Required | Rule |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `source` | string | Yes | built-in runtime source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` | | `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
| `artifact` | string | No | optional passthrough adapter field |
| `path` | string | No | optional passthrough adapter field |
| `required` | bool | No | optional input requirement | | `required` | bool | No | optional input requirement |
`artifact` and `path` are obsolete and rejected by strict configuration
loading. Use the canonical `source` identifier to select the input; Narratio
does not provide adapter-specific input passthrough fields.
Each `narratio.artifact.<name>` input source automatically contributes an
effective dependency on that configured artifact. Narratio unions these inferred
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
result during final configuration resolution. Authors use `depends_on` only for
ordering constraints that are not already expressed by an artifact input.
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
to inspect keys, definition sources, outputs, and effective dependencies without
loading a session or creating runtime state.
### Scriptorium Artifact Families
`pipeline.scriptorium.artifact_families` declares a shared artifact template
for every canonical campaign character. Configuration resolution expands each
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
planning or Scriptorium invocation. Party parsing rejects unsupported document
schemas before family expansion.
For each `pipeline.scriptorium.artifact_families.<name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `enabled`, `prompt_id`, `profile_id`, `timeout`, `render_debug`, `depends_on`, `inputs`, `vars` | ordinary artifact fields | No | copied to each generated artifact under the corresponding ordinary rules |
| `for_each` | string | Yes | exactly `party.characters` |
| `output_path_pattern` | string | Yes | safe path beneath `artifacts/` with exactly one `{character_id}` token and no other brace syntax |
| `member_vars` | map | No | maps an ordinary Scriptorium variable name to a supported canonical character selector |
| `member_dependencies` | list | No | unique family keys; each generated member depends on the corresponding generated member of each listed family |
| `publish` | map | No | typed family publish policy (`enabled`, `required`, `dest_pattern`) expanded into concrete publish outputs when enabled |
Generated keys are `<family>_<character_id>` and generated output paths must
not collide with explicit artifacts or another generated artifact. Families
expand even when disabled; normal analyze selection still omits disabled
artifacts unless they are explicitly selected by their concrete key.
Supported `member_vars` selectors are `character_id`, `player.name`,
`character.name`, `character.class_summary`, and `character.alias_summary`.
Their resolved values are strings. A member variable may not reuse a static
`vars` name; `session_id` remains owned and overwritten by Narratio as for any
other Scriptorium artifact.
Within a family only, an input source may use
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
infers the concrete member dependency. Use `member_dependencies` only for
additional same-character ordering constraints that are not already inputs.
This syntax is rejected in explicit artifacts and never reaches runtime stages
or Scriptorium.
### Notifications
Narratio currently supports only `notification.mode: noop`, which is also the
default when the section is omitted. The notify stage performs no delivery in
this mode. Backend, recipient, timeout, and other provider settings are
rejected by strict configuration loading until Narratio has a provider
integration.
### Campaign ### Campaign
| Field | Type | Required | Notes | | Field | Type | Required | Notes |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `campaign_id` | string | Yes | canonical campaign identity | | `campaign_id` | string | Yes | canonical opaque campaign identity |
| `session_template_file` | string | No | used by `session init` when set | | `session_template_file` | string | No | used by `session init` when set |
| `inputs.speakers_file` | string | Yes | stable input default | | `inputs.speakers_file` | string | Yes | stable input default |
| `inputs.autocorrect_file` | string | Yes | stable input default | | `inputs.autocorrect_file` | string | Yes | stable input default |
| `inputs.glossary_file` | string | Yes | stable input default | | `inputs.glossary_file` | string | Yes | stable input default |
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
### Session ### Session
| Field | Type | Required in session file | Notes | | Field | Type | Required in session file | Notes |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `session_id` | string | Yes | must match CLI session target when provided | | `session_id` | string | Yes | opaque identity; must match CLI session target when provided |
| `previous_session_id` | string | No | must not equal `session_id` | | `previous_session_id` | string | No | opaque identity; must not equal `session_id` |
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted | | `campaign` | string | No | opaque identity; filled from `campaign_id` during resolve if omitted |
| `date` | string | No | metadata | | `date` | string | No | metadata |
| `title` | string | No | metadata | | `title` | string | No | metadata |
| `inputs.speakers_file` | string | No | overrides campaign stable input | | `inputs.speakers_file` | string | No | overrides campaign stable input |
| `inputs.autocorrect_file` | string | No | overrides campaign stable input | | `inputs.autocorrect_file` | string | No | overrides campaign stable input |
| `inputs.glossary_file` | string | No | overrides campaign stable input | | `inputs.glossary_file` | string | No | overrides campaign stable input |
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
| `inputs.audio_dir` | string | Conditional | local audio mode | | `inputs.audio_dir` | string | Conditional | local audio mode |
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode | | `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode | | `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
@@ -263,13 +634,23 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
Audio rules: Audio rules:
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both. - configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
- `audio_s3` requires `pipeline.storage.backend: s3` and a configured S3 bucket.
### Storage backend selection
`local` is the default and disables remote object-store operations. Configure
`s3` explicitly before supplying `storage.s3`; a populated S3 block does not
select a backend on its own. Unknown backend names and an S3 block paired with
`local` are rejected during configuration validation.
### Previous-session expectation
`previous_session_id` is optional in a session file. When a command supplies
`--previous-session-id`, however, the session file must contain the same value;
an omitted or different value is rejected before the command performs work.
## Maintained Examples ## Maintained Examples
- `examples/pipeline.minimal.yml` See the [maintained examples index](../examples/README.md) for complete pipeline,
- `examples/pipeline.production.yml` campaign, session, template, and input fixtures. Keep complete copyable files
- `examples/pipeline.full.annotated.yml` there rather than duplicating them in this reference.
- `examples/campaigns/sample-campaign/campaign.yml`
- `examples/session.local-audio.yml`
- `examples/session.s3-audio.yml`
- `examples/session.template.yml`

64
docs/development.md Normal file
View File

@@ -0,0 +1,64 @@
# Development
This is the first-read landing page for people and LLM coding agents working on
Narratio. It provides a concise repository orientation and routes each kind of
change to its canonical documentation.
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
polished transcripts and generated artifacts. Start with the
[README](../README.md) for product context,
[Architecture](policy/architecture.md) for normative system boundaries, and the
[Internal Overview](internal/overview.md) for implemented component ownership.
## Go Prerequisite
Maintained development and verification require Go 1.26. Use the official Go
1.26 toolchain before running repository commands.
## What To Read
| When working on | Read | Why |
| --- | --- | --- |
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internal documents. |
| Application shape, boundaries, dependency direction, runtime invariants, safety properties, or dependencies | [Architecture](policy/architecture.md) | It defines the intended system shape, ownership, and non-goals. |
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical owners, audiences, current-behavior rules, and maintenance requirements. |
| Adding, changing, reviewing, rewriting, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and test lifecycle decisions. |
| CLI composition or command behavior | [Internal Overview](internal/overview.md) and [CLI Reference](cli.md) | The overview routes to command ownership; the reference owns public syntax and invocation behavior. |
| Configuration loading, resolution, or user-visible configuration | [Internal Overview](internal/overview.md) and [Configuration](config.md) | The overview routes to implementation ownership; the reference owns fields, defaults, discovery, and validation. |
| Session workflow, status, restore, cleanup, or object storage | [Restore Internals](internal/command-restore.md), [Workspace Internals](internal/workspace.md), [Storage Internals](internal/storage.md), [Operations](operations.md), and [Troubleshooting](troubleshooting.md) | These separate implementation mechanics, operator procedures, and symptom-driven recovery. |
| Pipeline sequencing or the behavior of a stage | [Internal Overview](internal/overview.md) and its focused stage documents | The overview owns the implemented stage inventory and routes to each stage contract. |
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
| Preparing, validating, or publishing a release | [Release Procedure](release.md) | The maintainer procedure owns version selection, candidate validation, guarded tag publication, and optional later CI inspection. |
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and package-level
contracts before changing behavior.
## Validation
Use focused package tests while iterating. Every pull request and push runs the
following repository-wide checks before it can be accepted:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go test ./internal/doccheck
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
```
The documentation check verifies local Markdown links and the dependency graph
of the Woodpecker workflows. The configuration check loads every maintained
pipeline and session example. Tag CI reuses this validation path before its
asynchronous asset publication; the maintainer release boundary is documented
in the [Release Procedure](release.md).
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
job to expose ordering and repeatability defects. Current runners cross-compile
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
target. The macOS cross-build establishes compilation only, not native runtime
evidence. Add a native check only when an official runner label and successful
native-run evidence are available.

View File

@@ -1,19 +1,36 @@
# Integrations Index # Integrations Index
## Audience ## Audience
Developers and coding agents changing Narratio's external integration boundaries.
Operators, developers, and coding agents who need to understand Narratio's
externally observable integration boundaries.
## Scope ## Scope
`docs/integrations/` is the implementation-level reference for downstream tool adapter contracts.
These docs cover what Narratio expects from external tools and what each adapter guarantees back to stage code. `docs/integrations/` is the canonical reference for protocols, invocation and
data contracts, logical outputs, and compatibility behavior at external tool
boundaries.
These documents describe what Narratio sends or invokes, what it accepts in
return, and how failures are surfaced. Internal composition and stage mechanics
belong in [the adapter implementation guide](../internal/adapters.md) and the
focused stage documents.
## Integration Contracts ## Integration Contracts
- `audita.md`: transcript polishing adapter (`audita process`).
- `seriatim.md`: merge/normalize/trim/render adapter (`seriatim`). - [Audita](./audita.md): transcript polishing (`audita process`).
- `scriptorium.md`: artifact run/render adapter (`scriptorium run|render`). - [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
discovery (`notarius run`).
- [Party](./party.md): canonical campaign roster input.
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
(`scriptorium run|render`).
- [WhisperX](./whisperx.md): speaker-audio transcription over HTTP.
## Related Canonical Docs ## Related Canonical Docs
- `docs/config.md`: operator-facing configuration reference.
- `docs/internal/adapters.md`: shared adapter boundary and runner wiring. - [Configuration](../config.md): operator-facing configuration reference.
- `docs/internal/stage-*.md`: stage-specific integration usage. - [Adapter implementation](../internal/adapters.md): shared adapter boundary and
runner wiring.
- [Internal documentation](../internal/overview.md): stage-specific integration
usage and component ownership.

View File

@@ -3,23 +3,24 @@
## Purpose ## Purpose
Define the Audita adapter contract used by the `polish` stage. Define the Audita adapter contract used by the `polish` stage.
## Adapter Boundary ## External Boundary
Interface:
- `audita.Runner`
- method: `Run(ctx, PolishRequest) (PolishResult, error)`
Primary implementation: Narratio invokes `audita process` as a subprocess for each polish operation.
- `internal/adapters/audita/SubprocessRunner` The configured timeout and parent cancellation bound the invocation. Internal
runner composition is documented in
Execution mode: [the adapter implementation guide](../internal/adapters.md).
- subprocess invocation of `audita process`
## Request Contract ## Request Contract
`PolishRequest` carries: `PolishRequest` carries:
- required transcript/glossary/output/work-dir paths; - required transcript/glossary/output/work-dir paths;
- optional report path (required when report mode is enabled); - optional report path (required when report mode is enabled);
- generated config and stdout/stderr log paths; - generated config and stdout/stderr log paths;
- optional module/model/base-url/config/output-schema/concurrency settings. - optional per-invocation module override.
The constructed runner owns static Audita settings: binary, timeout,
credentials, default modules, model and endpoint settings, validation and output
settings, report mode, and concurrency. The `polish` stage supplies only
invocation-specific paths and may override modules for that invocation.
## Result Contract ## Result Contract
`PolishResult` returns: `PolishResult` returns:
@@ -40,21 +41,29 @@ Construction fails for invalid static config values, including:
Run fails for: Run fails for:
- missing required request paths; - missing required request paths;
- missing required credential env var when configured (`llm_api_key_env`);
- subprocess execution failure; - subprocess execution failure;
- invalid processed transcript JSON (`segments` array required); - invalid processed transcript JSON (`segments` array required);
- invalid report JSON when reporting is enabled. - invalid report JSON when reporting is enabled.
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
MiB. Both must be regular files without symlinked path components.
Failure results still include output/log/config/exit metadata for diagnostics. Failure results still include output/log/config/exit metadata for diagnostics.
## Deterministic Behavior ## Deterministic Behavior
- CLI args are built from runner config + request in a fixed order. - CLI args are built from runner config + request in a fixed order.
- A configured typed `llm_api_key` reference is resolved during pipeline load;
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
override and is never re-read from the parent environment.
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested. - Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
- Manifest writes are stage-owned; adapter itself is stateless. - Manifest writes are stage-owned; adapter itself is stateless.
## Config Mapping ## Configuration
Config fields consumed through runner/stage wiring are under `pipeline.audita.*`.
Operator-selected values are defined under `pipeline.audita.*` in the
[configuration reference](../config.md#pipeline).
Maintained example with Audita config: Maintained example with Audita config:
- `examples/pipeline.full.annotated.yml`
- `examples/pipeline.production.yml` - [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -0,0 +1,200 @@
# Notarius Integration Contract
## Boundary
Narratio uses Notarius as a subprocess to extract configured structured JSON
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
Canonical Notarius v0.7.0 references:
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
records the exact current constraints for all ten D&D lanes. Treat the linked
Notarius documents as canonical when changing those values; Narratio does not
duplicate the complete schemas.
## Invocation
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
configuration path, input path, output directory, and working directory to
absolute paths. Configured references require support for the `--reference`
contract introduced in Notarius v0.6.0. Cache controls require support for the
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
cache integration baseline, not an unconditional minimum version; Narratio
does not impose a runtime version gate. Each reference binding is a separate
argument before `--json`. When `pipeline.notarius.config_profile` is present,
its exact value is another argument pair before `--json`:
```text
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
```
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
option requires a compatible Notarius executable, but deployments that omit the
field acquire no additional version requirement. Omitted or null configuration
adds no flag. Supplied empty, `default`, and named selectors are forwarded
without normalization so Notarius can apply its own ConfigKit semantics. This
option selects a Notarius configuration overlay, not a PromptKit profile through
`--llm-profile`, and Narratio never substitutes its own selected profile name.
A missing or invalid downstream overlay is an ordinary Notarius subprocess
failure: Narratio does not retry without the selector or fall back to another
profile.
Reference paths are absolute stable snapshots streamed from manifest-verified
canonical files prepared inside the current Narratio session workspace. They
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
full lowercase content digest. Narratio verifies snapshot checksum and size
before and after the subprocess, reconstructs missing or corrupt regular files
from a still-valid prepared source, and passes only configured bindings,
ordered lexically by normalized selector, as direct argument-vector entries
without shell interpretation. Symlinks, directories, other non-regular
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
binding takes precedence over a matching external path in Notarius
configuration. Narratio never emits `--without-reference`.
For canonical party configuration, the `party` binding is the unchanged,
validated authored roster and the `players` binding is its generated
projection. Both retain their established `narratio.input.party` and
`narratio.input.players` source IDs, and both are resolved from the prepared
manifest rather than from campaign configuration at extraction time.
The maintained D&D boundary binds only the four campaign-owned external slots:
```text
notarius run dnd-session \
--config <absolute config path> \
--input <absolute trimmed transcript path> \
--output-dir <absolute staging directory> \
--reference glossary=<absolute verified glossary snapshot> \
--reference party=<absolute verified party snapshot> \
--reference players=<absolute verified players snapshot> \
--reference spell_catalog=<absolute verified spell catalog snapshot> \
--json
```
The spell-catalog binding is omitted when the campaign does not maintain that
optional overlay. Registry, scene-description, combat-turn, and occurrence
handoffs generated during the same Notarius run remain in Notarius pipeline
composition and must not be emitted as CLI references. The linked CLI and D&D
consumer documents own selector targeting, declared slots, media compatibility,
and generated-handoff collision rules.
Standard output is reserved for the JSON receipt. Standard error is captured
separately as diagnostic output. Narratio applies the configured timeout and
does not interpret stdout as a receipt unless the subprocess exits successfully.
It does not pass a Narratio session ID or run `notarius config validate`
automatically; the configured working directory and Narratio's minimal child
environment apply to the subprocess.
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
Narratio neither reads nor synthesizes its configuration. Use explicit
persistent roots in Notarius when reuse must span subprocess invocations.
Changing a reference path/digest or other upstream identity input correctly
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
replacement for the active source; it does not delete unrelated plans or
checkpoint state.
When `pipeline.notarius.llm_api_key` is absent or null, Narratio adds no
credential to that environment. When configured, the field names an environment
variable whose already-loaded nonempty value is forwarded under that exact
name. Operators must use the same name as the `api_key_env` in the selected
Notarius/PromptKit profile. Narratio does not inspect the Notarius configuration
to infer a provider or destination name, and it does not forward unrelated
credentials from its parent environment or secrets directory.
The selected name is registered with Narratio's shared streaming redactor.
Credential bytes written by the child to captured stdout, captured stderr, or
stderr-derived failure diagnostics are replaced with `<redacted>`, including
custom names not known to the built-in sensitive-name list. Narratio does not
serialize the credential into arguments, stage metadata, manifests, or
generated configuration.
This captured-stream protection is not a filesystem sandbox. Notarius remains
responsible for keeping credentials out of files it writes directly, including
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
arbitrary downstream files and cannot promise redaction of transformed or
encoded credentials.
## Accepted Result
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
receipt remains `notarius.run-result.v2`; reference flags do not change the
receipt or ten-lane output contract. The receipt
must identify the configured pipeline, and its `index_file` must be exactly
`index.json` beneath the reported bundle root. The production index must name
the management files exactly as `manifest.json`, `rejected.json`,
`warnings.json`, and `diagnostics.json`. All receipt, index, and lane paths must
stay inside that bundle; symlinks and non-regular lane payloads are rejected.
Supported receipt and index shapes tolerate unknown fields for forward
compatibility, while required identity, validation, count, manifest,
rejection, warning, diagnostic, and lane-list fields remain mandatory.
Narratio applies bounded reads to the receipt, index, rejection, warning, and
diagnostic documents. Warning and diagnostic envelopes, group counts,
occurrence counts, truncation state, framework-owned origins, and
receipt-to-bundle counts must be internally consistent. Optional chunk-map and
evidence-context descriptors must carry their complete generic contract
metadata when present.
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
index descriptor with the configured lane ID, media type, schema ID, schema
version, and, when configured, module key. Missing, duplicate, rejected, or
incompatible required lanes fail extraction even if Notarius exited zero. A
configured lane whose v2 validation summary is `rejected` or `incomplete` also
fails extraction.
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
preserves that valid summary rather than treating it as a missing field.
Unconfigured lanes may remain in the preserved bundle but do not become
selectable Narratio sources.
Each accepted configured lane is registered as
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
resume validation but is not selectable. Scriptorium and publish rules consume
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
## Failure And Compatibility Behavior
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
- Unsafe or incompatible index data and required-lane rejection fail before the
staged bundle is promoted to durable storage.
- Contract and external provenance metadata are preserved on lane artifact
records and through explicit publication.
- Undeclared selectors, incompatible reference files, and external/generated
reference collisions are Notarius errors and fail extraction normally.
Rejection, validation, warning, and diagnostic summaries retain bounded stable
identity, category, origin, reason-code, status, and occurrence fields without
copying free-form external messages into Narratio manifest metadata or reading
lane payload bodies.
Configuration fields and defaults are in [Configuration](../config.md).
Operator paths, rerun procedures, and bundle retention are in
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
for failure recovery.
## Opt-in Integration Test
The production-boundary cache test requires an absolute path to a locally
built Notarius v0.7.0-compatible executable. It uses only a local
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
```sh
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
GOWORK=off go test ./internal/stage \
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
```
When the variable is unset, the normal test suite skips this test and never
searches `PATH`, a sibling checkout, the network, or a real provider.

View File

@@ -0,0 +1,71 @@
# Canonical Party Input
`party.yml` is a campaign-owned roster input. It must be a single, versioned
`narratio.party.v1` document when Narratio resolves a pipeline, campaign, and
session together.
```yaml
schema_version: narratio.party.v1
characters:
arannis:
player:
name: Eric
character:
name: Arannis
alias:
- Ari
- The Grey Owl
classes:
- name: wizard
level: 8
```
`characters` is a non-empty mapping. Each key is a stable character ID using
the configured-artifact key grammar: a lowercase ASCII letter followed by zero
or more lowercase ASCII letters, digits, or underscores. Character order is
preserved where roster order matters.
Every entry has `player.name`, `character.name`, and a non-empty
`character.classes` list. Class entries require a non-empty `name` and may
include a positive integer `level`. The optional, intentionally singular
`character.alias` field is a list. Names, aliases, and class names must be
non-empty, trimmed display strings without control characters. Character names
and aliases must be unique across the full roster under Unicode-aware
case-insensitive comparison; player names may repeat.
The document has exactly one YAML document and accepts no unknown fields.
`schema_version` is required: a missing, wrong, or malformed value is an error.
## Migration
Unversioned party inputs are rejected. Replace each legacy party document with
the versioned structure above, move each roster entry under `characters`, and
remove separate player and session-party settings: those keys are unsupported.
The configuration rules and source-relative path behavior are defined in the
[Configuration Reference](../config.md).
## Derived players document
During `prepare`, Narratio copies the canonical party source bytes unchanged
to `inputs/party.yml` and writes this deterministic players-only projection to
`inputs/players.yml`:
```yaml
schema_version: narratio.players.v1
players:
- name: Eric
character:
id: arannis
name: Arannis
alias:
- Ari
- The Grey Owl
```
There is one entry per character, sorted by stable character ID. Repeated
player names remain separate entries. The optional `alias` list retains its
declared order and is omitted when empty. The projection carries no class
data. Its prepared manifest record is marked `derived_from_party`; it is not a
separate user-provided `players_file`.

View File

@@ -3,20 +3,17 @@
## Purpose ## Purpose
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`. Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
## Adapter Boundary ## External Boundary
Interface:
- `scriptorium.Runner`
- methods:
- `RunArtifact(ctx, RunArtifactRequest)`
- `RenderArtifact(ctx, RenderArtifactRequest)`
Primary implementation: Narratio invokes Scriptorium as a subprocess in these modes:
- `internal/adapters/scriptorium/SubprocessRunner`
Execution modes:
- `scriptorium run` - `scriptorium run`
- `scriptorium render` - `scriptorium render`
The request timeout and parent cancellation bound each invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
## Request Contract ## Request Contract
Both request types carry: Both request types carry:
- binary/config/prompt/profile IDs; - binary/config/prompt/profile IDs;
@@ -49,18 +46,26 @@ Run behavior:
- `run` exit code `2` is mapped to `ValidationFailed=true`; - `run` exit code `2` is mapped to `ValidationFailed=true`;
- successful subprocess still fails if output file is missing or empty. - successful subprocess still fails if output file is missing or empty.
Each artifact result is limited to 64 MiB and must be a regular file without
symlinked path components.
Render behavior: Render behavior:
- subprocess errors propagate; - subprocess errors propagate;
- output file must exist and be non-empty. - output file must exist and be non-empty.
## Deterministic Behavior ## Deterministic Behavior
- input and var maps are sorted into deterministic `--input` and `--var` CLI args. - input and var maps are sorted into deterministic `--input` and `--var` CLI args.
- stage wiring adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream routing, overriding any configured `vars.session_id`.
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested. - generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
- adapter is stateless and does not own artifact-selection policy. - adapter is stateless and does not own artifact-selection policy.
## Config Mapping ## Configuration
Config fields consumed through runner/stage wiring are under `pipeline.scriptorium.*` plus per-artifact settings under `pipeline.scriptorium.artifacts.*`.
Operator-selected values are defined under `pipeline.scriptorium.*`, including
per-artifact settings under `pipeline.scriptorium.artifacts.*`, in the
[configuration reference](../config.md#pipeline).
Maintained examples with Scriptorium config: Maintained examples with Scriptorium config:
- `examples/pipeline.full.annotated.yml`
- `examples/pipeline.production.yml` - [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -3,24 +3,19 @@
## Purpose ## Purpose
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`. Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
## Adapter Boundary ## External Boundary
Interface:
- `seriatim.Runner`
- methods:
- `Run(ctx, MergeRequest)`
- `Normalize(ctx, NormalizeRequest)`
- `Trim(ctx, TrimRequest)`
- `Render(ctx, RenderRequest)`
Primary implementation: Narratio invokes Seriatim as a subprocess in these modes:
- `internal/adapters/seriatim/SubprocessRunner`
Execution modes:
- `seriatim merge` - `seriatim merge`
- `seriatim normalize` - `seriatim normalize`
- `seriatim trim` - `seriatim trim`
- `seriatim render` - `seriatim render`
The configured timeout and parent cancellation bound each invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
## Request/Result Contracts ## Request/Result Contracts
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report. - `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema. - `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
@@ -46,6 +41,8 @@ Invocation fails on:
- empty render output files. - empty render output files.
When report paths are provided/enabled, report files must parse as JSON. When report paths are provided/enabled, report files must parse as JSON.
Each Seriatim JSON or rendered-text result is limited to 64 MiB and must be a
regular file without symlinked path components.
## Deterministic Behavior ## Deterministic Behavior
- argument ordering is deterministic per command construction. - argument ordering is deterministic per command construction.
@@ -53,9 +50,13 @@ When report paths are provided/enabled, report files must parse as JSON.
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested. - generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
- adapter does not write manifests or choose stage inputs. - adapter does not write manifests or choose stage inputs.
## Config Mapping ## Configuration
Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*` and `pipeline.render.*`.
Operator-selected values are defined under `pipeline.seriatim.*` and
`pipeline.render.*` in the
[configuration reference](../config.md#pipeline).
Maintained examples with Seriatim config: Maintained examples with Seriatim config:
- `examples/pipeline.full.annotated.yml`
- `examples/pipeline.production.yml` - [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)

View File

@@ -0,0 +1,92 @@
# Integration: WhisperX
## Purpose
WhisperX transcribes each prepared speaker audio file for Narratio's
`transcribe` stage. Narratio uses an HTTP boundary and installs each successful
response as that speaker's raw transcript JSON.
## HTTP Boundary
Narratio sends an HTTP `POST` to the configured transcription URL using
`multipart/form-data` with:
- `file`: the audio file, retaining its base filename; and
- `language`: the configured language string.
The server must return a `2xx` response whose body is valid JSON. Narratio does
not currently require a more specific response schema at this boundary.
The transcription URL must be an absolute `http` or `https` URL. Each
application attempt creates one bounded multipart framing and streams the audio
from an open file between that framing; it does not buffer the complete audio
or spool a multipart copy. The file field precedes the language field.
WhisperX response acquisition is capped at 10 MiB.
The initial request body and any transport-requested replay use the same
multipart boundary and framing. Every body opens an independent audio handle
and replay starts at the beginning. Narratio closes all bodies created for the
attempt on response, failure, timeout, or cancellation, including when a
transport abandons an earlier body. Genuine source read/close failures remain
request failures; deliberate early closure of an abandoned original does not
poison a valid replay.
## Request And Result Contract
Each adapter request identifies a speaker, a readable audio file, and the
destination for the raw transcript. The HTTP request carries the audio and
language; the speaker identifier remains Narratio orchestration metadata.
On success, Narratio atomically writes the response body to the requested
destination. The adapter result reports that logical output together with the
attempt count, final HTTP status when available, elapsed duration, and adapter
identity metadata. A failed or invalid response is not installed as the
transcript output.
## Retry, Timeout, And Cancellation
- The configured timeout applies independently to each HTTP attempt.
- `retries` means additional attempts after the first.
- HTTP `429`, HTTP `5xx`, attempt timeouts, and network errors are retryable.
- Other HTTP `4xx` responses and explicit cancellation are not retryable.
- Narratio waits the configured retry delay between attempts and aborts that
wait when the parent context is canceled.
Transport replay occurs inside one application attempt and does not consume an
entry from the configured retry budget or increment the reported attempt
count. The standard HTTP redirect policy remains in effect; body replay can
also support body-preserving redirects.
## Validation And Failure Semantics
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
a missing language, a non-positive timeout, negative retries, or a negative
retry delay. A request fails before transmission when its audio or output path
is missing.
Non-`2xx` status, transport failure, response-size overflow, invalid JSON, or
failure to install the output causes the transcription to fail. Errors include
attempt context, and the result retains attempts, final status when available,
and elapsed duration for diagnostics.
Replay makes a fresh request body available to the HTTP transport; it does not
provide an exactly-once server guarantee and does not prove recovery from every
proxy or protocol-level failure. A server may have received work before the
client learns that a request failed. WhisperX operations should therefore be
safe to retry for the same input when practical.
## Determinism And Concurrency
Each audio request has stable multipart field names, and successful bytes are
installed atomically. The transcribe stage may process speaker files in
parallel, bounded by the configured concurrency. Each validated speaker is
checkpointed independently; any speaker failure still fails the aggregate
stage, while committed earlier speakers remain reusable on an ordinary retry.
## Related Canonical Docs
- [Configuration](../config.md#pipeline) defines the operator-selected
WhisperX URL, language, timeouts, retry policy, and concurrency.
- [Adapter implementation](../internal/adapters.md) describes internal wiring.
- [Transcribe stage](../internal/stage-transcribe.md) describes stage mechanics,
durable artifacts, and manifests.

View File

@@ -1,45 +0,0 @@
# Internal Documentation Index
## Audience
Developers and coding agents changing Narratio internals.
## Scope
`docs/internal/` documents implemented internal contracts: stage boundaries, manifest/state behavior, artifact resolution, restore behavior, storage boundaries, and workspace invariants.
User and operator behavior belongs in:
- `docs/cli.md`
- `docs/config.md`
- `docs/operations.md`
- `docs/troubleshooting.md`
## Pipeline Stage Set
Canonical stage order from `internal/stage.All()`:
1. `prepare`
2. `transcribe`
3. `merge`
4. `polish`
5. `normalize`
6. `trim`
7. `render`
8. `analyze`
9. `publish`
10. `notify` (placeholder)
`notify` is currently a placeholder stage with optional notifier call behavior; it has no persisted pipeline outputs.
## Internal Component Docs
- `adapters.md`: external adapter boundaries and default runtime wiring.
- `artifacts.md`: canonical source IDs, runtime catalog behavior, and resolution rules.
- `manifest.md`: session and run manifest contracts.
- `storage.md`: object-store interface and S3 implementation behavior.
- `workspace.md`: local session layout, run-local layout, and cleanup guardrails.
- `command-restore.md`: restore discovery, planning, execution, and reporting.
- `stage-prepare.md`
- `stage-transcribe.md`
- `stage-merge.md`
- `stage-polish.md`
- `stage-normalize.md`
- `stage-trim.md`
- `stage-render.md`
- `stage-analyze.md`
- `stage-publish.md`

View File

@@ -1,49 +1,121 @@
# Internal: Adapters # Internal: Adapters
## Purpose ## Purpose
Define external integration boundaries and default adapter wiring used by app/stage orchestration.
Explain the adapter interfaces and production composition used by application
and stage orchestration. Externally observable protocols and formats belong in
the [integration contracts](../integrations/).
## Adapter Boundaries ## Adapter Boundaries
Narratio stage logic depends on adapter interfaces, not transport-specific details. Narratio stage logic depends on adapter interfaces, not transport-specific details.
Primary adapters: Primary adapters:
- `whisperx.Client` - `whisperx.Client`
- `seriatim.Runner` - `seriatim.Runner`
- `audita.Runner` - `audita.Runner`
- `scriptorium.Runner` - `scriptorium.Runner`
- `notarius.Runner`
- `storage.ObjectStore` - `storage.ObjectStore`
- `notify.Sender` - `notify.Sender`
## Ownership ## Ownership
Adapters own: Adapters own:
- HTTP/subprocess/SDK argument and transport details. - HTTP/subprocess/SDK argument and transport details.
- Backend-specific request/response mapping. - Backend-specific request/response mapping.
Adapters do not own: Adapters do not own:
- stage ordering/skip/force logic; - stage ordering/skip/force logic;
- manifest transitions; - manifest transitions;
- canonical path policy. - canonical path policy.
## Default Wiring ## Default Wiring
`internal/app/runner.go` initializes default adapters when not injected:
- WhisperX HTTP client from pipeline config. The application composes default adapters per invocation when they were not
- Seriatim subprocess runner. injected through the stage environment, and only when the selected execution
- Audita subprocess runner. plan needs them:
- Scriptorium subprocess runner.
- Noop notifier (`notify.NoopSender`). - WhisperX HTTP client for `transcribe`.
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
- Audita subprocess runner for `polish`.
- Scriptorium subprocess runner for `trim` or `analyze`.
- Notarius subprocess runner for `extract` when extraction is enabled.
- Noop notifier (`notify.NoopSender`) for `notify`.
- Object store only when required by selected stages/config. - Object store only when required by selected stages/config.
Object-store construction goes through `newCommandObjectStore`, which loads configured filesystem secrets before adapter initialization. Remote publish locks are loaded only for a selected, enabled publish that
uploads a run. Shared session lifecycle setup still applies to every selected
range, but an unselected integration is neither initialized nor validated by
application-owned composition. Constructor, clock, randomness, remote-lock,
and post-publish deletion boundaries are scoped to the invocation; stages still
receive their collaborators through the stage environment. Each selected stage
retains its own fail-fast configuration and input validation.
`session plan` is outside production adapter composition. It performs
resume validation and models selected transitions against cloned manifest
state without constructing or invoking stage-execution adapters. The shared
command configuration loader may still use object storage to retrieve a missing
remote session file before planning begins.
Notarius is composed only when extraction is enabled; the extract stage owns
prepared reference resolution, receipt, bundle, and configured-lane policy.
The adapter validates the ordered selector/absolute-path pairs and is the sole
owner of serializing them as repeated `--reference` arguments before `--json`.
It also serializes an effective resume request as `--resume` and a nonempty
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
adapter, resolves force precedence before constructing the request.
Its request may also carry one already-resolved opaque ConfigKit secret
reference. Only the Notarius adapter reads that value, rejects an unresolved or
empty programmatic reference, and maps the exact configured name/value into the
shared subprocess override and sensitive-name fields. It never rereads the
parent environment. A nil reference preserves the previous invocation.
Application-owned object-store construction loads configured filesystem
secrets before invoking the invocation's storage factory. Production commands
use the concrete storage adapter constructor; focused tests can supply an
in-memory backend without changing package state.
## Failure Semantics ## Failure Semantics
- Constructor errors fail stage execution setup early. - Constructor errors fail stage execution setup early.
- Runtime adapter errors propagate to stage code and then manifest failure handling. - Runtime adapter errors propagate to stage code and then manifest failure handling.
- Subprocess adapters persist stage logs/generated configs through stage-managed paths. - Subprocess adapters persist stage logs/generated configs through stage-managed paths.
- Shared subprocess execution keeps private lifecycle mechanics separate from
Narratio's environment, diagnostics, redaction, and error policy. It starts
an owned process group on Linux and macOS. Every terminal path disposes of
that owned tree before returning. After a natural leader exit, the runner checks
for remaining group members and uses bounded graceful then forceful termination.
Cancellation, deadlines, and
diagnostic limits use the same terminal disposal path without losing their
original result classification. Child environments contain only the execution
baseline and adapter-specified values; configured credentials are resolved
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
departed leader's descendant. Unsupported platforms reject owned command
execution.
## Test Surfaces ## Implementation And Tests
- Composition: `internal/app/runner_composition.go`,
`internal/app/runner_dependencies.go`, `internal/app/object_store.go`
- Shared subprocess mechanics: `internal/adapters/subprocess`
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
- `internal/adapters/whisperx/http_test.go` - `internal/adapters/whisperx/http_test.go`
- `internal/adapters/seriatim/subprocess_test.go` - `internal/adapters/seriatim/subprocess_test.go`
- `internal/adapters/audita/subprocess_test.go` - `internal/adapters/audita/subprocess_test.go`
- `internal/adapters/scriptorium/subprocess_test.go` - `internal/adapters/scriptorium/subprocess_test.go`
- `internal/adapters/notarius/subprocess_test.go`
- `internal/adapters/storage/*_test.go` - `internal/adapters/storage/*_test.go`
- `internal/app/runner_test.go` - `internal/app/runner_test.go`
See the [WhisperX](../integrations/whisperx.md),
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
[Scriptorium](../integrations/scriptorium.md), and
[Notarius](../integrations/notarius.md) contracts before changing an
externally visible boundary. Operator-selected values belong in
[Configuration](../config.md).

View File

@@ -1,38 +1,92 @@
# Internal: Artifacts # Internal: Artifacts
## Purpose ## Purpose
Define canonical artifact IDs, runtime catalog behavior, source resolution rules, and shared current-state mechanics used by app and previous-cache code.
Explain the artifact registry, runtime catalog, resolver, previous-input
requirements, and shared remote current-state mechanics implemented by
`internal/artifacts`. Configuration fields that accept source IDs belong in
[Configuration](../config.md); physical placement belongs in
[Operations](../operations.md).
## Built-in Source IDs ## Built-in Source IDs
- `narratio.transcript.base` -> `transcripts/base.json` (`merge`) The internal registry recognizes these stable built-in source IDs:
- `narratio.transcript.polished` -> `transcripts/polished.json` (`polish`)
- `narratio.transcript.final` -> `transcripts/final.json` (`normalize`)
- `narratio.transcript.final_trimmed` -> `transcripts/final.trimmed.json` (`trim`)
- `narratio.transcript.final_markdown` -> `transcripts/final.md` (`render`)
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` (`render`)
- `narratio.bounds.session` -> `artifacts/session_bounds.json` (`trim`)
## Configured and Previous-Session Sources - `narratio.transcript.base`
- `narratio.transcript.polished`
- `narratio.transcript.final`
- `narratio.transcript.final_trimmed`
- `narratio.transcript.final_markdown`
- `narratio.transcript.final_trimmed_markdown`
- `narratio.bounds.session`
Registry entries bind each ID to its producer, output kind, and content
validator. The focused stage documents own their input/output flow;
[Configuration](../config.md) owns where operators may select these IDs.
## Configured, Extraction, And Previous-Session Sources
- configured source ID format: `narratio.artifact.<artifact_key>` - configured source ID format: `narratio.artifact.<artifact_key>`
- extraction source ID format: `narratio.extraction.<output_key>`
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>` - previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
Both formats are validated by strict source-policy rules. All formats are validated by strict source-policy rules. Configured artifact and
extraction keys use `^[a-z][a-z0-9_]*$`; source parsers never normalize an
unrecognized token into a valid source. Extraction sources are registered only
from `pipeline.notarius.outputs`; the Notarius index has no selectable source
ID.
Prepared stable source IDs are `narratio.input.players`,
`narratio.input.party`, `narratio.input.glossary`, and
`narratio.input.spell_catalog`. Artifact policy owns their canonical manifest
kind and prepared filename vocabulary. Canonical party mode preserves the
party source bytes in the party record and supplies the players record from the
deterministic `derived_from_party` projection; both remain ordinary prepared
source IDs for consumers.
## Runtime Catalog ## Runtime Catalog
`ArtifactCatalog` tracks: `ArtifactCatalog` tracks:
- `planned`: source registered for run context; - `planned`: source registered for run context;
- `executable`: selected and enabled for analyze execution; - `executable`: included in the effective analyze artifact set;
- `available`: local file exists and validates; - `available`: the source's canonical evidence owner validates its current
manifest record and durable bytes;
- `provenance`: availability source. - `provenance`: availability source.
Configured definitions are always registered. Without an explicit selection,
the effective analyze set contains enabled definitions. With `--artifacts`, the
exact named configured definitions become the effective set for that invocation,
regardless of their `enabled` value. The effective-set resolver itself does not
expand dependencies. Configuration has already materialized the sorted union of
explicit ordering edges and configured-artifact input edges, and the analyze
work planner closes targets over that one concrete prerequisite graph.
Availability is separate from executability. Configuration may normalize a
family selection into its concrete generated members before this resolver runs.
The effective set retains optional family
and character origin metadata, but its keys, catalog sources, and runtime
lookups remain concrete configured-artifact identities.
Family publish policies are likewise expanded into ordinary configured-source
publish rules during configuration resolution.
Configured outputs, including non-executable prerequisites, become available
only when the versioned analyze state identifies a current result whose source,
contract, canonical configured path, size, and checksum match a confined
no-follow regular file. An incidental canonical file is unavailable, and a
manifest with an unsupported aggregate-only representation is rejected before
catalog construction.
Extraction entries are registered from configuration and become available only
after compatible extraction evidence is hydrated.
During an analyze invocation, a newly validated and atomically materialized
configured output is marked available with its producer run ID, contract,
checksum, and size. Later scheduled dependents therefore observe the same
semantic identity whether their prerequisite was reused from current manifest
evidence or produced earlier in the invocation.
Current provenance values: Current provenance values:
- `generated.current_analyze_run` - `generated.current_analyze_run`
- `filesystem.disabled_artifact_output` - `manifest.current_analyze_artifact`
- `manifest.inputs.previous_cache` - `manifest.inputs.previous_cache`
- `current_session.previous_cache` - `current_session.previous_cache`
@@ -40,18 +94,65 @@ Current provenance values:
Built-ins: Built-ins:
1. manifest producer outputs (when present) - resolve only from the current manifest's succeeded registered producer;
2. canonical session-path fallback - require exactly one output with the registered source ID and output kind, a
safe session-rooted path, producer run ID, canonical checksum, and valid
content; and
- verify the registered checksum against the durable bytes. Incidental files,
canonical filenames, duplicate outputs, failed/stale producer records, and
malformed evidence are unavailable.
Configured sources (`narratio.artifact.*`): Configured sources (`narratio.artifact.*`):
- resolve only through runtime catalog availability. - resolve only through runtime catalog availability;
- use the shared typed analyze-evidence inspection in
`analyze_evidence.go` for prior current-session results;
- require the supported analyze-state and fingerprint versions, a `current`
record for the exact configured key and source ID, a complete contract, the
configured canonical relative path, positive stored size, and stored
checksum matching bytes read from a confined no-follow regular file; and
- treat non-current statuses, malformed records, removed keys,
unsafe or missing files, and size/checksum mismatches as unavailable without
rewriting manifest state. Catalog construction iterates current
configuration, so removed or renamed records are not advertised.
`narratio.member_artifact.*` is not a runtime source family. Configuration
resolution accepts it only in an artifact-family declaration and rewrites it
to the corresponding configured source before this catalog is built.
Prepared stable sources (`narratio.input.*`):
- resolve only from the current manifest's exact prepared-input record;
- require the policy-owned canonical path below the session root, a confined
non-symlink regular file, a non-empty payload, and a matching SHA-256
checksum; and
- return an immutable source/path/checksum/size identity shared by extract and
analyze rather than falling back to campaign/session source paths.
Extraction sources (`narratio.extraction.*`):
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
- require a current successful extract record with the exact configured source,
compatible contract and Notarius provenance, a confined regular durable
payload, matching checksum, and the current resolved trimmed-transcript
identity;
- remain unavailable unless catalog hydration receives valid evidence. Resume
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
an error; and
- are never inferred by scanning the Notarius bundle directory.
Previous-session sources (`narratio.previous_session.artifact.*`): Previous-session sources (`narratio.previous_session.artifact.*`):
- resolve only from local `previous/` cache state; - resolve only from one matching `previous_artifact` input record in the
- prefer manifest-backed previous-input paths; current manifest, including its exact source ID and checksum;
- fallback to existing previous-cache filesystem paths. - are hydrated only from the unique `published_output` declaration for their
canonical source in the selected remote commit; and
- never infer a path from a previous manifest, current configuration, basename,
or existing cache file.
Source absence is evaluated by the consuming artifact input. An optional input
is omitted from that invocation; a required input fails resolution. This is
separate from a stage's lifecycle outcome.
Validation by content type: Validation by content type:
@@ -64,7 +165,7 @@ Validation by content type:
`CollectPreviousArtifactRequirements`: `CollectPreviousArtifactRequirements`:
- scans enabled configured artifacts only; - scans the effective configured artifact set;
- extracts only canonical previous-session sources; - extracts only canonical previous-session sources;
- deduplicates by artifact key; - deduplicates by artifact key;
- merges required and optional references (required wins); - merges required and optional references (required wins);
@@ -72,19 +173,47 @@ Validation by content type:
## Current-State Helpers ## Current-State Helpers
Artifacts package owns shared remote current-state loading mechanics used by restore, status/validate checks, and previous-cache planning. Artifacts package owns shared remote current-state loading mechanics used by
restore, status and validation checks, and previous-cache planning.
The selected session manifest is accepted only with
`schema_version: narratio.session-manifest.v1` and complete stage identity.
An unsupported local or downloaded manifest must be replaced by regeneration,
verified restore, or republication; artifact resolution never fills in missing
authority markers.
For a new-protocol current state, the pointer-selected immutable commit is the
complete restore authority. Callers receive its declared object identities and
must not supplement them by listing mutable session prefixes.
A `published_output` declaration must use a canonical publish-compatible source
ID and an exact `runs/{run_id}/published/{destination}` key in the same session.
Only this declaration type may refer to an earlier run; other declared objects
remain confined to the selected commit's run.
The reader opens each small control object directly and enforces owner-specific
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
immutable commit manifest, and 8 MiB for the selected session manifest. These
are exposed as `MaxCurrentCommitPointerBytes`,
`MaxRemoteCommitManifestBytes`, and `MaxRemoteSessionManifestBytes`.
Each read uses the generation and size metadata returned with its opened body.
Actual bytes remain subject to a limit-plus-one read even if size metadata is
absent or inaccurate. Immutable selections then retain their declared-size,
checksum, generation, and identity checks. No current-state control object is
downloaded through a temporary file. Artifact and previous-session downloads
validate their complete sibling temporary file against available committed
identity before replacing a canonical local path.
Core helpers: Core helpers:
- `LoadCurrentRunPointer`
- `LoadCurrentManifest`
- `LoadCurrentState` - `LoadCurrentState`
- `ValidateCurrentStateIdentity` - `ValidateCurrentStateIdentity`
- `RemoteCommitManifest` and `CurrentCommitPointer`
Typed missing-state errors: Typed missing-state error:
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`) - `CurrentCommitPointerMissingError` (`ErrCurrentCommitPointerMissing`)
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
Identity validation supports caller-provided expectations: Identity validation supports caller-provided expectations:
@@ -98,6 +227,21 @@ Caller policy is intentionally outside artifacts helpers:
- some callers downgrade missing state to status/findings; - some callers downgrade missing state to status/findings;
- some callers skip optional behavior when state is missing. - some callers skip optional behavior when state is missing.
## Remote Lock Objects
The artifacts package defines the independent remote-lock payload and identity
contract. Each canonical publish source maps to one object below the validated
session `locks/` prefix; its filename is the lowercase SHA-256 digest of the
exact source followed by `.lock`.
Lock payloads are strict versioned JSON containing the canonical source, an
optional reason, and a UTC creation time. Reads are bounded to
`MaxRemoteLockObjectBytes` (64 KiB), reject unknown fields and trailing values,
and verify that the payload source matches the key digest. Exact lookup reports
presence explicitly, prefix listing validates every object and sorts by source,
creation uses an absence precondition, forced replacement uses one observed
generation, and removal is idempotent.
## Key Path Helpers ## Key Path Helpers
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for: `internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
@@ -107,9 +251,52 @@ Caller policy is intentionally outside artifacts helpers:
- spool/cache paths; - spool/cache paths;
- S3 session/run/current-state key layout. - S3 session/run/current-state key layout.
New publication creates run-scoped immutable objects, including
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
mutable selector is `current/commit-pointer.json`; readers verify its selected
commit and declared object generations/checksums.
A `published_output` declaration may identify its commit's run or an earlier
immutable run in the same campaign/session namespace. The key parser validates
that namespace, rejects mutable and malformed keys, and derives the originating
run from the exact declaration. All other commit artifacts remain confined to
the commit's own run prefix.
Campaign, session, and Narratio run IDs are validated as portable opaque
segments at configuration and artifact boundaries before they can be used in a
workspace or S3 namespace. Previous-artifact destinations remain typed,
multi-segment relative paths and are confined beneath `previous/artifacts`; they
are not treated as opaque identifiers.
See [Workspace Internals](workspace.md) for how callers consume local helpers
and [Operations](../operations.md#local-state-layout) for the authoritative
physical layout.
## Invariants ## Invariants
- source ID formats are stable contracts; - source ID formats are stable contracts;
- artifact resolution is deterministic and manifest-aware; - artifact resolution is deterministic and manifest-aware;
- extraction sources are available only from a compatible successful manifest
record;
- previous-session source resolution in `analyze` is local-only; - previous-session source resolution in `analyze` is local-only;
- remote current-state key construction remains centralized in artifacts helpers. - remote current-state key construction remains centralized in artifacts helpers.
## Implementation And Tests
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
`internal/artifacts/extraction_catalog.go`,
`internal/artifacts/extraction_evidence.go`,
`internal/artifacts/extraction_input.go`,
`internal/artifacts/prepared_input.go`
- Current state: `internal/artifacts/current_state.go`,
`internal/artifacts/current_state_commit.go`,
- Paths and keys: `internal/artifacts/paths.go`,
`internal/artifacts/s3_keys.go`
- Previous requirements: `internal/artifacts/previous_requirements.go`
- Tests: `internal/artifacts/artifact_resolver_test.go`,
`internal/artifacts/catalog_test.go`,
`internal/artifacts/extraction_catalog_test.go`,
`internal/artifacts/current_state_test.go`,
`internal/artifacts/paths_model_test.go`,
`internal/artifacts/previous_requirements_test.go`

View File

@@ -1,24 +1,29 @@
# Internal: Command Restore # Internal: Command Restore
## Purpose ## Purpose
Define the implemented `narratio session restore` command contract:
- committed remote current-state discovery; Explain the implemented restore discovery, planning, installation, and
- deterministic restore planning; reporting flow in `internal/app`. User invocation belongs in
- safe local install semantics; [CLI](../cli.md#session-restore), and the operator recovery procedure and
- durable restore reporting. physical restore scope belong in
[Operations](../operations.md#restore-workflow).
Restore separates remote authority, local conflict policy, and filesystem
mutation so each remains testable independently.
The public command supplies production defaults to one private,
invocation-scoped dependency value. Discovery, planning, and execution use that
same value, so focused sequencing tests can replace one boundary without
mutable package hooks.
## Discovery Contract ## Discovery Contract
Restore resolves remote committed state from the session publish current pointers: Discovery delegates current-state pointer and manifest loading to
`internal/artifacts`, then validates the result against the resolved request:
- `current/run_id.txt` (required, non-empty);
- `current/manifest.json` (required, decodable).
Current-state discovery uses shared artifacts-level mechanics and validates identity against the resolved request config:
- campaign must match; - campaign must match;
- session ID must match. - session ID must match.
- run ID must match the pointer-selected committed run.
Restore treats any missing or invalid remote current state as a command error. Restore treats any missing or invalid remote current state as a command error.
@@ -32,28 +37,27 @@ Restore planner action kinds:
Planner behavior: Planner behavior:
- remote list scope is the resolved session prefix; - a new-protocol restore uses only the selected commit's declared artifact set;
each action carries that artifact's immutable key, checksum, size, and
generation;
- remote-to-local mapping is traversal-safe; - remote-to-local mapping is traversal-safe;
- actions are sorted deterministically by local relative path. - actions are sorted by local relative path and then remote key;
- planning opens the selected session root and inspects every target by its
normalized session-relative path. Missing ancestors produce a missing target;
an ancestor or leaf symlink is a non-regular conflict and is never opened for
hashing;
- force converts differing eligible regular files from conflicts to downloads;
directories, symlinks, and other non-regular targets remain conflicts.
Restore scope from current remote state: For a non-dry-run restore, planning/classification happens only after acquiring
the session lock. Runner manifest/reuse checks acquire that same lock first.
- include `manifest.json`; Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
- include `transcripts/**`; prepare, status, and validation. Each requirement selects the unique
- include `artifacts/**`; `published_output` for its canonical source in the pointer-selected commit,
- include `audio/**` only with `--include-audio`. then strips only that run's immutable `published/` prefix for its safe local
cache path. It never probes candidate objects or derives paths from manifests,
Explicit exclusions from current remote state mapping: configuration, or basenames.
- `current/**`;
- `runs/**`;
- `logs/**`;
- `reports/**`;
- `config/**`;
- `inputs/**`;
- `previous/**`.
Previous-cache files are planned separately through `previouscache.BuildPlan` when configured previous-session requirements exist.
## Execution Contract ## Execution Contract
@@ -61,24 +65,63 @@ Execution order and safety:
- non-manifest downloads happen before manifest install; - non-manifest downloads happen before manifest install;
- `manifest.json` installs last; - `manifest.json` installs last;
- downloads use sibling temp files plus atomic rename; - downloads use sibling temp files, validate the complete object, then use an
atomic rename;
- manifest replacement is validated before rename; - manifest replacement is validated before rename;
- each committed object is verified against its declared checksum, size, and
generation before installation;
- installation reopens the selected session root, creates only validated
missing directory components with workspace modes, and rejects symlinked or
non-regular ancestors and leaves before replacement;
- a committed manifest already verified during discovery is retained for the
matching restore action and revalidated before installation, avoiding a
second body transfer;
- failed installs do not roll back files already written in the same execution. - failed installs do not roll back files already written in the same execution.
- a durable `.restore-incomplete.json` marker is written before installation.
It blocks runners until a restore retry completes all verified installs and
the local manifest replacement, at which point it is removed.
- restored manifest local references are rebased beneath the selected local
session root. Unsafe relative references and producer-machine absolute paths
outside the manifest's producer session root are rejected; producer-local
spool paths and run-local diagnostic references are not restored as
authority. Cleanup remains ordinary restored stage state and is not reset or
synthesized by restore.
Audio restore path: Audio restore path:
- uses `audio.MaterializeS3Audio`; - uses `audio.MaterializeS3Audio`;
- integrates spool and S3 audio cache paths; - integrates spool and S3 audio cache paths;
- supports cache-hit reuse without object redownload. - reuses cached audio only when its opened regular file, content digest, and
identity sidecar all match the selected remote object version; otherwise it
refreshes through the validated download path.
## Reporting Contract ## Reporting Contract
- `--dry-run`: prints summary only; no local writes. - dry-run mode prints a summary, performs no durable session writes, and may
- non-dry-run: writes `reports/restore-latest.json`. read remote current-state or object-identity data to produce that summary;
- execution mode persists the canonical restore report described in
[Operations](../operations.md#restore-workflow);
- report includes plan counts, per-action status, and execution failures. - report includes plan counts, per-action status, and execution failures.
## Invariants ## Invariants
- restore uses committed remote current state as authority; - restore uses committed remote current state as authority;
- `current/run_id.txt` is the remote publish commit marker; - one restore or status inspection observes the single pointer-selected commit
loaded at discovery; later pointer changes cannot add objects or substitute a
different run into its plan;
- a verified `current/commit-pointer.json` and its selected immutable commit
establish remote commitment; when absent, republish the session;
- restore does not execute pipeline stages. - restore does not execute pipeline stages.
## Implementation And Tests
- Discovery: `internal/app/restore_discovery.go`
- Planning: `internal/app/restore_plan.go`, `internal/previouscache`
- Execution: `internal/app/restore_execute.go`
- Reporting and command coordination: `internal/app/restore_report.go`,
`internal/app/restore.go`
- Tests: `internal/app/restore_discovery_test.go`,
`internal/app/restore_plan_test.go`,
`internal/app/restore_execution_test.go`,
`internal/app/restore_recovery_test.go`,
`internal/app/restore_workflow_test.go`

View File

@@ -0,0 +1,215 @@
# Configuration Internals
User-visible fields, defaults, and selection behavior belong in the
[Configuration Reference](../config.md). This document describes the internal
pipeline-loading boundary implemented by `internal/config`.
## Pipeline Loading
`LoadPipeline` assembles and validates a pipeline in this order:
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
enabled imports, and the caller's profile selection. Omitted, empty, and
exact `default` selections use the base; named profiles use the fixed
`profiles/<name>.yml` convention.
2. ConfigKit parses the root and declared imports, additively merges base
sources, applies only the selected overlay, and strictly decodes the result.
3. The definition resolver applies recorded entry-level and inherited defaults,
runs deterministic domain normalization, records derived publish destinations,
and resolves ordinary paths relative to the root pipeline file.
4. The definition validator performs read-only checks that require only the
pipeline file. Checks requiring canonical-party family expansion remain at
the existing explicit pipeline/campaign resolution boundary.
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
declared by multiple base files, adapts source ownership records, attaches
runtime-only state, and computes the normalized effective digest.
This ordering preserves monolithic configuration behavior. Moving a field to
an imported fragment changes its source ownership, not its path base, default,
or schema semantics.
## Loaded Context Resolution
`LoadedPipelineCampaign` carries one already composed pipeline and its selected
campaign into session resolution. `LoadSessionWithPipelineCampaignOptions`
loads a local session against that context, while
`ResolveLoadedPipelineCampaign` also accepts an already loaded remote session
or no session while a caller retrieves one. Compatibility loaders route through
these functions after their initial pipeline and campaign reads.
Application commands own pipeline and campaign discovery, campaign-file versus
registry selection, and the corresponding mutual-exclusion rules. Once they
have a `LoadedPipelineCampaign`, local session discovery and remote-session
download retain that exact pipeline object and its private provenance. Removing
a temporary downloaded session file therefore cannot invalidate the resolved
pipeline or campaign context.
Campaign and session files use independent ConfigKit definitions with imports
and profiles disabled. Campaign files normally load directly from their source;
session files are first read through the 8 MiB bounded-read boundary because the
application must reject unresolved template placeholders before passing an exact
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
the same typed loader and retain the existing post-load identity expectations,
stable-input precedence, and source-relative path semantics.
The application also has a separate read-only inspection resolver for `config
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
campaign selection functions, but never routes through session discovery,
remote-session download, adapter composition, workspace
initialization, manifest access, or cleanup. A pipeline with retained artifact
family declarations must resolve its selected campaign before ordinary pipeline
validation, which expands its canonical-party members and generated publish
rules. A pipeline without those declarations may be validated by itself.
Secret-directory bootstrap remains invocation-scoped and happens before the
ordinary ConfigKit load because explicit typed references must resolve even for
inspection.
`MarshalEffectivePipeline` is the configuration-owned projection for `config
show`. It serializes one explicit typed safe projection with the existing YAML
encoder. The projection has no slots for resolution-only artifact family
declarations or private provenance. The result has one trailing newline;
commands do not marshal runtime objects directly.
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
safe provenance projection for `config sources`. Pipeline ownership begins with
the complete logical field paths projected from the schema and classifies
each contributor as root, import, profile, or centralized default. The
projection replaces generated concrete member paths with paired family and
canonical-party records, and does the same for generated publish rules.
Campaign records identify campaign-owned fields and party inputs; canonical
derived players point to the party source. The application command only joins these sorted
records with selection metadata and never reparses configuration files.
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
`config artifacts`. It derives source identity from each map key, carries sorted
effective dependencies, aggregates ConfigKit definition contributors, and adds
family/character/party provenance for generated members. It does not inspect
files named after artifacts or infer another runtime graph.
`config diff` uses a thin paired loader that performs two independent ConfigKit
loads. When campaign
resolution is needed, the command loads one selected campaign and party and
expands both effective pipelines from that same party value. The configuration
owner projects each normalized effective mapping directly from that same typed
projection into sorted logical paths using JSON number preservation;
mapping leaves are compared individually while sequence values remain atomic.
Values are compact deterministic JSON representations for command output, not
raw YAML fragments, ownership records, or secret material. A differing digest
with no projected difference is treated as an internal consistency error.
Campaign context construction reads the campaign-owned party source through
`ParseParty`. The required canonical party retains its raw bytes and normalized
roster in runtime-only `ResolvedParty` provenance. Canonical resolution creates
a deterministic `derived_from_party` players projection. The party is the only
configured roster authority; player and session-party fields are unsupported.
## Canonical Party Domain
`ParseParty` is the package-owned boundary for parsing a party source. Its
single-source ConfigKit definition disables imports and profiles, requires
`schema_version` through `VersionRule`, and strictly decodes the
character-keyed typed schema before applying Narratio's party-domain checks.
Successful metadata coordinates recover authored character-map order without a
second YAML parse; class and alias slice order comes directly from typed decode.
The canonical value retains a separate exact byte copy of its source so
consumers can materialize the authored party document without reserializing it.
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
players-only projection.
An unversioned source is rejected before resolution, family expansion, or
prepare-stage materialization.
## Validation Ownership
Configuration validation remains within `internal/config`, with the public
entry points and pipeline-domain sequencing separated from the private owners
for core pipeline/session rules, external adapters, publish/storage, Notarius,
Scriptorium, and shared scalar/path checks. These are package-local ownership
boundaries rather than extension points or a general validation framework.
Validation is read-only, fails fast, and does not define which of several
independent invalid fields is reported first. Map-backed declarations are
nevertheless traversed deterministically. Reusable domain normalization is an
explicit preceding operation for both loaded and programmatically constructed
pipelines. Notarius normalization commits canonical scalar, reference, and output
values only after the complete normalization succeeds; its validator then checks
that candidate without changing it. Scriptorium validates artifact declarations
and outputs, dependencies and inputs, variables, referenced outputs, and the final
dependency graph in deterministic artifact order.
Diagnostics identify the relevant semantic field. Merge conflicts also
retain every claiming source needed to correct an additive ownership conflict.
Complete prose, punctuation, formatting, and private validation choreography are
not compatibility contracts.
## Diagnostics And Runtime Metadata
Syntax, duplicate-key, merge, profile, and schema failures include the
relevant source name and full field path. Additive conflicts report every
claiming source so operators can repair the split without repeatedly
rediscovering additional conflicts.
The loaded pipeline retains private runtime metadata for the absolute root
path, ordered imports, selected profile name and selection source, selected
overlay, original ConfigKit metadata and source snapshots, effective digest,
and leaf ownership. Base
leaves retain their root/import owners, replaced leaves belong to the selected
overlay, and centrally supplied values use the synthetic `default` owner. This
metadata does not participate in YAML decoding or alter the public
configuration model.
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
execution, but the digest substitutes their normalized logical values captured
before root-relative resolution, so relocating an equivalent configuration
bundle does not change provenance. Because ConfigKit and resolution metadata
are private, the digest excludes source layout, profile name, and ownership.
Configuration stores credentials as ConfigKit typed references. A successful
load resolves their values inside the opaque typed boundary, while this safe
projection serializes only `EnvName()` identities. Credential-value changes are
therefore never hashed; reference-name changes remain ordinary provenance.
The optional Notarius credential follows the same path even for disabled or
resumable extraction; normalization copies the opaque reference without reading
or resolving it again. Direct YAML serialization has a configuration-owned safe
projection so only the reference name can leave this boundary.
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
for later runtime expansion.
## Test Surfaces
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
base selection, unselected-overlay isolation, source attribution, and Narratio's
unique artifact/family base-declaration rule. Existing focused configuration
tests protect defaults, explicit-value semantics, strict decoding, validation,
metadata, and root-relative path behavior. Application configuration-loader tests protect the
single-read boundary by changing the pipeline file after its initial load and
confirming local session resolution retains the original pipeline. Other
configuration tests continue to protect defaults and validation after assembly.
`party_test.go` protects the versioned typed party schema, domain invariants,
authored character order, retained bytes, and deterministic players projection
without involving campaign or runtime wiring.
`party_resolution_test.go` protects campaign-owned party loading, canonical
input restrictions, source provenance, and virtual players input selection.
## Artifact Family Resolution
Pipeline loading retains `scriptorium.artifact_families` as a resolution-only
declaration. Once campaign party resolution establishes a canonical roster,
configuration expands families in sorted family-key and character-ID order
into ordinary `ScriptoriumArtifactConfig` values. The expansion owns the narrow
`{character_id}` output substitution, closed member-variable selectors, key and
output collision checks, and the runtime-only family-origin catalog. It then
removes family declarations from `ScriptoriumConfig`, runs ordinary Scriptorium
validation, and refreshes the effective pipeline digest. Stages and adapters
therefore receive only concrete artifact maps.
The catalog retains sorted family member keys plus family/character/source
origins and the typed dependency/publish declarations for their later owners.
`member_dependencies` add corresponding ordinary concrete dependencies, while
the family-only `narratio.member_artifact.<family>` input form is rewritten to
the matching ordinary configured-artifact source. The catalog records those
resolved dependency and input identities with their declaring family and party
member. No member-artifact source is registered as a runtime policy source.
An enabled family publish declaration expands to ordinary configured-artifact
publish rules before the existing publish and lock validators run. Runtime
publication consequently receives no family wildcard or special matcher.

55
docs/internal/fileops.md Normal file
View File

@@ -0,0 +1,55 @@
# Internal: File Operations
`internal/fileops` owns the narrow mechanics for atomically visible replacement of one
byte file. Callers keep ownership of serialization, validation, cancellation,
and destination-directory policy.
## Destination Paths
Ordinary workspace replacement uses standard operating-system path semantics.
Callers with a root-scoped safety contract own that validation at their boundary.
Remote object acquisition is owned by its domain caller. The caller writes a
sibling temporary file, validates the complete result, and passes that file to
`InstallSiblingTempFile` only when it is ready to become canonical. Callers own
remote object selection, validation, conflict handling, and temporary-file
cleanup.
Directory bundles are not promoted by `internal/fileops`; extraction owns its
validated Notarius bundle rename.
## Cleanup Contract
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant, opens
that root with `os.OpenRoot`, and removes the normalized root-relative target
with `Root.RemoveAll`. It rejects root, traversal, and out-of-root targets; a
missing target is successful. `RemoveFileUnderRoot` additionally requires an
existing leaf to be regular. Command and post-publish policy remains owned by
`internal/app`.
## Bounded Reads
`ReadFileBounded` opens a normal path and reads at most its caller-supplied
limit. Credential mode policy and environment precedence remain owned by
`internal/app`.
## Replacement Contract
`ReplaceFileAtomic` requires an existing destination directory. It creates a
sibling temporary file, writes the complete byte sequence, applies the
caller-supplied mode, closes the file, runs an optional pre-rename check, and
replaces the destination with a rename.
The pre-rename check is the last point at which a caller can cancel without
installing a new destination. A failure before the rename leaves the old
destination unchanged and removes the temporary file. Successful replacement
provides atomic visibility; it makes no crash- or power-loss durability claim.
Replacement follows the operating system's same-filesystem rename semantics.
If a platform cannot replace an existing destination, the operation returns an
error and never removes the old file as an emulation step.
`WriteFileAtomic` and the copy helpers create the destination parent with the
repository's workspace permissions. `InstallSiblingTempFile` requires the
caller to provide an existing temporary file in the existing destination
directory.

View File

@@ -1,21 +1,47 @@
# Internal: Manifest # Internal: Manifest
## Purpose ## Purpose
Define durable session state (`manifest.json`) and invocation state (`runs/{run_id}/manifest.json`) contracts.
Explain the session-progress model implemented by `internal/manifest`. Physical
manifest placement belongs in
[Operations](../operations.md#local-state-layout).
## Session Manifest ## Session Manifest
Path:
- `{workspace.root}/work/{campaign}/{session_id}/manifest.json`
Primary model (`manifest.Manifest`): `manifest.Manifest` records:
- required `schema_version: narratio.session-manifest.v1`
- identity (`session_id`, `campaign`, `run_id`) - identity (`session_id`, `campaign`, `run_id`)
- local path metadata (`local_workdir`, `local_spool_dir`) - local path metadata (`local_workdir`, `local_spool_dir`)
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`) - remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
- `inputs` records - `inputs` records
- durable `artifacts` records - durable `artifacts` records
- per-stage `stages` map - per-stage `stages` map, including the ordinary cleanup stage record
Session, campaign, and run identities in local and downloaded manifests must be
portable opaque segments. Unsafe legacy identities are rejected with migration
guidance rather than being normalized into a different workspace or remote
namespace.
Decoding is strict at every typed manifest field. Unknown fields are rejected;
the schema version remains `narratio.session-manifest.v1` and there is no
compatibility reader that silently drops retired state. Cleanup progress uses
only the cleanup entry in `stages`.
Every persisted session manifest has a non-nil `stages` map. Each entry is
non-nil and its embedded `name` exactly matches its map key; loading never
invents either marker. A missing or unsupported schema version, or incomplete
stage identity, is unsupported local state: move it aside and regenerate it,
or restore a verified current committed snapshot when that is authoritative.
Prepare records independent `party` and `players` input checksums. In canonical
party mode, the party record retains its campaign source identity while the
players record uses `derived_from_party`; raw roster content is never embedded
in manifest metadata. Both records remain the durable authority for consumers
of their prepared input source IDs.
The model admits these stage states:
Stage status enum:
- `pending` - `pending`
- `running` - `running`
- `succeeded` - `succeeded`
@@ -24,34 +50,306 @@ Stage status enum:
- `stale` - `stale`
- `interrupted` - `interrupted`
## Run Manifest ### Transcribe-owned speaker state
Path:
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`
Run model (`manifest.RunManifest`): A `transcribe` stage record may carry `transcribe_state_version: 1` and a
- invocation identity and `force` flag `transcribe_speakers` map keyed by the existing derived speaker ID. The version
- requested stages is the authority marker: a versioned empty map means that no earlier speaker
- per-stage action (`run` or `skip`) completion is reusable. Absence of both fields is accepted as legacy
- per-stage status aggregate-only state so current Narratio can read older manifests.
- overall run status (`running`, `succeeded`, `failed`)
Each map entry represents one completed speaker only. It binds a versioned
fingerprint to the session-relative audio path, audio SHA-256 and size, one
canonical `transcript_raw` output with its path, SHA-256 and size, and the
producer run ID. Paths must be canonical and session-relative, checksums must
be lowercase SHA-256 values, and the record and output producer identities
must agree. Other stages cannot carry this transcribe-owned state.
Audio size is nonnegative; WhisperX determines whether that audio can be
transcribed. Output size remains positive and the transcript must be valid JSON.
Transcribe reconciles this collection against the current prepared audio set
and observable WhisperX semantics. Missing or corrupt regular outputs become
work; unsafe or non-regular evidence is rejected. It saves the reconciled
collection before HTTP dispatch and saves a detached candidate after each
validated completion. Only after a save succeeds is the candidate installed as
live authority. Canonical bytes produced before an unsuccessful save are not a
completion record.
Aggregate transcribe status still describes the whole invocation. A failed,
running, stale, or interrupted aggregate may therefore retain reusable speaker
records, while aggregate outputs are declared only after the current
collection is complete. Ordinary retry reconciles and reuses those records.
Force first checkpoints an empty current collection, preventing later retries
from reviving pre-force results.
Older aggregate-only transcribe success is conservatively regenerated once
when transcribe is selected because it does not bind individual outputs to the
audio bytes used. The session-manifest schema identifier remains
`narratio.session-manifest.v1`; this additive state is accepted by current
Narratio, but older strict-decoding binaries may reject manifests after these
fields have been written.
### Analyze-owned artifact state
Every `analyze` stage record carries `analyze_state_version: 1` and an
`analyze_artifacts` map keyed by normalized configured artifact key. The
version is the authority marker: version 1 with no entries is a valid evaluated
empty set. An absent or unsupported version is unsupported local state and is
rejected rather than being treated as aggregate-only authority.
Each analyze artifact record has one disposition:
- `current`: the configured artifact is available and carries a versioned
fingerprint plus a complete output record and separate output size;
- `stale`: the recorded semantic identity is no longer current;
- `missing`: no validated current result exists;
- `failed`: the attempted work failed and carries a bounded diagnostic; or
- `unselected`: the artifact was intentionally outside the evaluated set.
Records bind their normalized key and dependencies, fingerprint contract when
evaluated, canonical session-relative output identity when current, producing
Narratio run, update time, and bounded non-secret Scriptorium provenance and
diagnostic paths. A current output includes its configured source ID, contract,
checksum, and positive byte size. Non-current records cannot carry an output,
so an older file is not advertised through stale, missing, failed, or
unselected state.
Family-produced records additionally retain optional `family` and
`character_id` provenance supplied by configuration resolution. These fields
do not replace the concrete configured key or infer family membership from a
name, so older records without them remain valid.
The session-stage collection is the reconciled authority across invocations.
These records remain analyze-owned data inside the fixed stage; they are not
dynamic stages or generic subtasks.
Analyze checkpoints one complete validated candidate collection after each
successful artifact and before exposing that artifact to later work. It saves a
detached manifest candidate and installs its version, collection, and updated
timestamp into live state only after the store reports success. Aggregate stage
outputs are ordinary result outputs rebuilt in configured-key order from current
session records only.
Successful incremental execution replaces only evaluated artifact records and
preserves valid unrelated current records. Rebuilt outputs are compared by
bytes and contract: an unchanged identity permits an unselected dependent with
the same recomputed fingerprint to remain current, while a changed identity
removes output authority from every unselected transitive dependent by marking
it stale. A partial analyze invocation can therefore succeed while unrelated
configured records remain stale. Existing canonical files never create current
records without validated execution and a successful checkpoint.
Aggregate analyze status is deliberately coarser than this collection. Resume
validation may skip a succeeded aggregate record when the selected artifact
closure is current even if unrelated records are stale. Conversely, a stale
aggregate record may cross the ordinary runner boundary and perform zero
Scriptorium calls when reconciliation proves every selected artifact current;
ordinary successful stage completion then restores the aggregate status.
On an artifact error, analyze installs a complete retry collection on the live
manifest before returning. The active record is failed without output, current
transitive dependents are stale, unrelated current records survive, and only
earlier checkpointed completions remain current. The runner's ordinary failure
path marks aggregate analyze failed, clears aggregate outputs, invalidates
delivery dependents, and saves that collection. A terminal session-save failure
is joined with the original adapter, validation, filesystem, cancellation, or
checkpoint cause.
If an artifact checkpoint fails, analyze rolls its working collection back to
the last committed collection before constructing the active failed record.
Newly materialized canonical bytes therefore remain incidental and cannot
become current authority. A later ordinary run reconciles the durable records,
reuses earlier current artifacts, and resumes with the failed or stale work.
The same recovery applies after process interruption: the aggregate record may
be non-successful, but every completed artifact checkpoint remains reusable.
## Remote Commit Manifest
`artifacts.RemoteCommitManifest` is a separate, versioned remote snapshot
contract. It is not a serialized session manifest and contains no local
post-publication assertion such as `current_pointer_written`. A remote commit
identifies one campaign, session, and run and declares its immutable artifact
set. Each artifact has a typed source, immutable destination key, SHA-256
checksum, size, and storage generation.
`current/commit-pointer.json` is the sole mutable selector for the new
contract. It identifies exactly one run-scoped `runs/{run_id}/commit.json` and
binds that object by checksum, size, and generation. Readers strictly reject
unknown fields, version mismatches, pointer/commit identity mismatches, and
objects that do not match their declaration.
An absent commit pointer means no remote state is published. Republish to
establish a current immutable snapshot.
## Persistence Semantics ## Persistence Semantics
`manifest.LocalStore`: `manifest.LocalStore`:
- validates loaded documents;
- normalizes missing maps/stage records; - accepts only the exact supported session-manifest schema version;
- writes atomically via temp file + rename; - rejects unknown fields and multiple JSON values;
- validates stage-map identity without synthesizing missing maps, records, or
names;
- retains only representation-preserving collection normalization;
- writes through a sibling temporary file and atomic replacement visibility;
- updates `updated_at` on save. - updates `updated_at` on save.
## Execution Semantics Manifest replacement provides atomic visibility, not a crash- or power-loss
Runner updates both manifests per stage transition: durability guarantee. Callers reload after a returned save error before retrying.
- mark running A transcribe or analyze checkpoint error leaves the prior manifest collection
- mark succeeded/failed/skipped as authority. The runner then attempts to persist the aggregate failure; if
- persist logs/generated config refs and metadata that terminal save also fails, its error is joined with the originating failure
and the next invocation must reconcile from the last readable manifest.
Session manifest is the authoritative stage-progress ledger across invocations. ## Execution Semantics
Run manifest is invocation-scoped audit state.
Planning and execution share one pure pre-execution decision over the recorded
stage status, force selection, and normalized resume-validation result. That
decision identifies run versus reuse and any required stale and dependent
invalidation transition. Planning applies it only to a cloned manifest.
Execution remains the sole owner of observed stage results and persistence.
For each selected stage, the application runner computes semantic evidence,
performs any eligible resume validation, applies the pre-execution decision,
and persists required invalidation before either reusing the existing success
or beginning an attempt. The same one-stage execution owner marks the attempt
running before invoking the stage and then records and persists its failed,
skipped, or successful terminal state before a later stage may start. On
success it records outputs, logs, generated configuration references, metadata,
and—when the stage implements the optional contract—a versioned
semantic-configuration fingerprint. Artifact
records may include optional contract and external provenance objects; old
manifests remain compatible when those fields are absent. A successful forced
rerun marks only succeeded transitive dependent session-stage records stale.
The application owns a fixed dependency relation distinct from execution order;
dependents are returned in canonical order. Render and extract therefore never
stale one another, while either can stale analyze, publish, notify, and cleanup.
Starting an execution clears the current session-stage record's prior outputs,
logs, generated configuration references, metadata, and semantic fingerprint.
Failed and skipped
transitions enforce the same clearing rule directly, while success repopulates
only fields returned by the new result. Marking a record stale does not clear
those details because resume validation and diagnosis may still require them
before execution begins.
Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
speaker collection and the analyze-owned per-artifact collection. This lets
later reconciliation replace only non-current entries without erasing durable
completions. Other stages retain aggregate-only lifecycle behavior and are
forbidden from carrying either collection.
A stage may explicitly return a skipped disposition and stable reason. The
runner persists that outcome in the session manifest, clears older outputs,
logs, generated configuration references, and metadata, then applies any
bounded details from the current skip and continues. This self-skip is distinct from deciding not to execute an
already-succeeded stage and is reconsidered on later runs. Skipped results
cannot contain outputs. An intentional self-skip records the current semantic
fingerprint because it is a completed, reusable stage result; failed or
interrupted work never promotes one.
When an already-succeeded stage is skipped, the session manifest deliberately
retains its existing succeeded record because it remains the cross-invocation
progress authority. If a stage supplies semantic configuration evidence, reuse first
requires the persisted positive schema version and lowercase SHA-256 digest to
match the current resolved stage semantics. Missing semantic evidence, malformed
evidence, or a mismatch makes the stage and its fixed transitive dependents
stale. The existing stage-specific resume
validator runs only after this semantic check succeeds; both checks are
required. Extraction, analyze, and cleanup have resume validators and may
reject an otherwise eligible skip when their selected durable evidence is
obsolete; the runner marks the aggregate record stale and executes it.
Analyze's validator can still accept a partial selection when only unrelated
artifact records are stale.
Implemented reuse coverage is deliberately split between aggregate semantic
evidence and focused durable validators:
| Work | Reuse authority | Focused owners |
| --- | --- | --- |
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
| transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
| trim | aggregate semantic fingerprint | [trim](stage-trim.md), [Scriptorium](../integrations/scriptorium.md), [Seriatim](../integrations/seriatim.md) |
| render | aggregate semantic fingerprint | [render](stage-render.md), [Seriatim](../integrations/seriatim.md) |
| extract | aggregate semantic fingerprint plus reference/output validator | [extract](stage-extract.md), [Notarius](../integrations/notarius.md) |
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
| cleanup | aggregate semantic fingerprint plus publish-evidence and target-absence validator | [cleanup](stage-cleanup.md), [workspace](workspace.md) |
These contracts record resolved choices Narratio can observe, not operational
runner tuning. External model, module, prompt, profile, and configuration-file
contents that a tool privately loads remain outside the contract when their
configured identifier is unchanged; operators must force the affected work
after such a private content change.
The session manifest is the authoritative stage-progress ledger across
invocations. It retains the most recently resolved invocation's bounded
configuration provenance. It identifies the selected profile name and source
(`default` or `cli`) plus the effective configuration digest, but never a raw
secret or profile content. This provenance is informational: it does not
participate in stage resume or cache decisions. A profile change therefore
invalidates only stages whose semantic configuration changed. When a private
external-tool model, module, prompt, or profile changes behind an unchanged
configured identifier, use `--force` for the affected work.
`session plan` computes the same current fingerprint and applies the shared
decision and invalidation rules to a cloned manifest. It predicts the runner
decision without persisting session state. The semantic fingerprint owner
hashes deterministic JSON from stage-owned typed structs; stage providers must
exclude secrets, complete effective-configuration dumps, and operational
values that cannot affect canonical results. Concrete coverage is owned by the
focused stage and integration documents linked above.
Before an explicitly bounded execution starts after `prepare`, the application
reads the session manifest and accepts only `succeeded` or `skipped` for every
excluded canonical prefix stage. The first other status or absent record fails
the request before layout mutation, adapter initialization, session-manifest
writes, or run-directory creation. Excluded prefix records are not passed to
resume validators. Records after the selected end are not prerequisites and
may be made stale by selected work without being scheduled.
After a publish commits remotely, any configured local cleanup is first recorded
as a session-manifest obligation before deletion begins. Each target becomes
complete only after its confined deletion (or safe absence check) and a
successful manifest save. An incomplete obligation is retried when publish
executes again and retains the committed run and remote identity that authorized
it; an invocation that does not execute publish does not perform cleanup.
Each invocation derives campaign, session, run, local-path, and remote-prefix
metadata from the validated resolved configuration as one projection. A persisted
session manifest must agree on campaign and session identity before execution;
the current projection is refreshed for every invocation while stage progress,
inputs, and durable artifacts remain session history.
For handled failures after the session identity transition, the runner records
the failure on the session ledger and atomically attempts its save while
preserving the originating error. Abrupt process death remains an accepted case
where the last session transition can require operator interpretation.
## Invariants ## Invariants
- stage resume/skip decisions are session-manifest driven. - stage resume/skip decisions are session-manifest driven.
- force reruns stale downstream succeeded stages. - semantic fingerprint comparison precedes stage-specific resume validation.
- run manifest does not replace session manifest as progress authority. - only successful and intentional-skipped results promote current semantic
evidence.
- running, failed, and self-skipped stages do not retain result payloads from
an earlier success.
- stale stages retain prior details until replacement execution starts.
- force reruns stale succeeded stages in the fixed dependency relation.
- the session manifest is the only durable local progress ledger.
- remote commitment is established by a verified current pointer and remote
commit relationship, never by a mutable session-manifest boolean.
## Implementation And Tests
- Models and transitions: `internal/manifest/manifest.go`
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
`internal/artifacts/current_state_commit.go`,
- Persistence and validation: `internal/manifest/store.go`
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
- Package tests: `internal/manifest/*_test.go`
- Assembled execution behavior: `internal/app/runner_test.go`,
`internal/app/run_stage_test.go`

140
docs/internal/overview.md Normal file
View File

@@ -0,0 +1,140 @@
# Internal Overview
This document is the implemented component map for Narratio. Normative system
boundaries and dependency direction belong in
[Architecture](../policy/architecture.md). User and operator contracts belong
in the [CLI](../cli.md), [Configuration](../config.md),
[Operations](../operations.md), and [Troubleshooting](../troubleshooting.md).
Externally observable tool and format contracts belong under
[Integrations](../integrations/).
## Execution Path
```text
cmd/narratio -> internal/app -> configuration and production composition
-> internal/stage -> adapters and external systems
-> manifests and artifact resolution -> durable local/remote output
```
The executable delegates process behavior to the application boundary. The
application resolves configuration, composes concrete collaborators, acquires
session safety controls, and runs commands. Pipeline commands execute the
canonical stage sequence through adapter interfaces, while manifests record
progress and artifact services resolve durable inputs and outputs.
The pipeline runner follows one explicit invocation lifecycle: inspect bounded
prerequisites without mutation, prepare the workspace layout, acquire one
session lock, load or create the authoritative session manifest, compose only
the collaborators needed by the selected range, execute each selected stage in
order, and then return the summary. Cleanup, when selected, is the ordinary
final stage rather than runner finalization work.
The lock remains held from manifest initialization through finalization, and a
failed stage or required manifest save prevents later stages from starting.
## Components
| Area | Implemented owners | Responsibility |
| --- | --- | --- |
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus atomic single-file replacement, bounded reads, workspace modes, and root-scoped cleanup helpers. |
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
The application boundary composes concrete implementations. Stages depend on
Narratio-level contracts; external transport and SDK details remain in
adapters. The normative rules for these relationships remain in
[Architecture](../policy/architecture.md).
Command execution normalizes one private dependency value per invocation.
Pipeline configuration-path discovery, configuration reads, secret-directory
reads, remote downloads, object-store construction, and restore phase functions
use that value throughout the command. Public command entry points supply the
canonical discovery and production defaults, while focused tests may provide
invocation-owned discovery candidates or replace only the external or sequencing
boundary they exercise without mutating package state.
Pipeline execution and `session plan` share the same inclusive contiguous-range
model and the same pure pre-execution decision over recorded status, force, and
normalized resume validation. Planning applies that decision and modeled
selected-stage transitions to cloned session state; it does not persist, create
invocation state, initialize stage-execution adapters, or claim to observe an
execution result. Command configuration loading can still retrieve a
missing session file through configured remote storage. It retains the initially
composed pipeline and selected campaign while resolving either a local or
downloaded remote session, so one invocation cannot mix pipeline revisions.
Analyze planning additionally exposes the artifact closure's targets,
prerequisite rebuilds, execution order, and current reuse.
## Pipeline Stage Set
The implemented canonical order is:
1. [`prepare`](stage-prepare.md)
2. [`transcribe`](stage-transcribe.md)
3. [`merge`](stage-merge.md)
4. [`polish`](stage-polish.md)
5. [`normalize`](stage-normalize.md)
6. [`trim`](stage-trim.md)
7. [`render`](stage-render.md)
8. [`extract`](stage-extract.md)
9. [`analyze`](stage-analyze.md)
10. [`publish`](stage-publish.md)
11. `notify` (no-op)
12. [`cleanup`](stage-cleanup.md)
`notify` currently has no persisted pipeline outputs and uses the explicit
`noop` notification mode. Its versioned semantic evidence records that delivery
mode and excludes adapter credentials and response data. The focused stage
documents own implementation mechanics. The
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
and execution semantics.
Execution order and invalidation are separate application contracts. The stage
registry owns the flat execution sequence. The application orchestration owner
uses a fixed, validated dependency relation to find transitive dependents in
canonical order. In particular, `render` and `extract` are sibling consumers of
trimmed transcript state: neither invalidates the other, while either can stale
`analyze`, `publish`, `notify`, and `cleanup`. Cleanup is a dependent of notify,
so any earlier replacement can stale a prior cleanup result.
## Focused Documentation
- [Configuration Internals](configuration.md): pipeline imports and profiles,
field ownership, decoding, and root-relative path semantics.
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
failure behavior, and test surfaces.
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
resolution, previous requirements, and current-state helpers.
- [Manifest Internals](manifest.md): session records, persistence, and execution
transitions.
- [Storage Internals](storage.md): object-store interface and S3 behavior.
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
guardrails.
- [Restore Internals](command-restore.md): discovery, planning, execution, and
reporting.
- [`prepare`](stage-prepare.md)
- [`transcribe`](stage-transcribe.md)
- [`merge`](stage-merge.md)
- [`polish`](stage-polish.md)
- [`normalize`](stage-normalize.md)
- [`trim`](stage-trim.md)
- [`render`](stage-render.md)
- [`extract`](stage-extract.md)
- [`analyze`](stage-analyze.md)
- [`publish`](stage-publish.md)
- [`cleanup`](stage-cleanup.md)
Use this map to find an owner, then read its focused documentation and tests
before changing behavior.
The stage registry is implemented in `internal/stage/placeholders.go` and its
ordering is protected by `internal/app/planner_test.go`. Cross-invocation skip,
force, failure, and invalidation behavior is exercised in
`internal/app/runner_test.go` and `internal/app/run_stage_test.go`.

View File

@@ -1,39 +1,210 @@
# Stage: analyze # Stage: analyze
## Purpose ## Purpose
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
Reconcile configured Scriptorium artifacts, execute only required work in
dependency order, and safely materialize validated outputs.
## Inputs ## Inputs
- configured artifacts from `pipeline.scriptorium.artifacts`
- optional selected artifact filter (`--artifacts`) - ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
- built-in/configured/previous-session source references in artifact inputs party artifact families have already expanded into this map during
configuration resolution, including corresponding member dependencies and
rewritten member-artifact input sources. Each artifact's dependency list is
already the normalized union of explicit ordering edges and configured-artifact
input edges.
- optional selected artifact keys supplied through the stage environment
- built-in, configured, extraction, and previous-session source references in
artifact inputs
Supported source families: Supported source families:
- built-ins: `narratio.transcript.*`, `narratio.bounds.session` - built-ins: `narratio.transcript.*`, `narratio.bounds.session`
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
`narratio.input.glossary`, `narratio.input.spell_catalog`
- configured artifacts: `narratio.artifact.<key>` - configured artifacts: `narratio.artifact.<key>`
- extraction lanes: `narratio.extraction.<key>`
- previous-session cache: `narratio.previous_session.artifact.<key>` - previous-session cache: `narratio.previous_session.artifact.<key>`
## Outputs ## Outputs
- one materialized output per executed configured artifact (`output_path`)
- one current per-artifact manifest record per validated materialized output
- stage metadata describing selected/generated/reused artifacts - stage metadata describing selected/generated/reused artifacts
## Key Behavior ## Key Behavior
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
- builds runtime artifact catalog (built-ins + configured artifacts). - when `pipeline.scriptorium` is absent or no configured artifact is
- marks non-executable configured artifacts as reusable when output files already exist. executable, completes successfully and records explanatory metadata. Before
returning, it reconciles and checkpoints the complete session collection:
records removed from configuration are dropped, while valid records for
configured but unselected artifacts remain authoritative. This is not an
explicit self-skip: the session manifest records success, satisfies publish's
prerequisite, and an ordinary later run reuses the result while the effective
set remains empty. Enabling or selecting an artifact later makes missing
versioned evidence non-resumable and schedules it without requiring force.
- builds a runtime artifact catalog containing built-ins, configured artifacts,
and configured extraction lanes. Extraction availability is hydrated only
from compatible successful extraction evidence.
- uses enabled configured artifacts by default. An explicit `--artifacts`
selection is a one-invocation override that makes exactly the named
configured artifacts explicit targets even when disabled. The work planner
adds required configured prerequisites, reuses current ones, and schedules
stale, missing, or otherwise non-current prerequisites before dependents.
- makes a non-executable configured artifact reusable only when its current
manifest record and durable output pass the configured-artifact evidence
contract; an incidental or stale canonical file is unavailable.
- validates selected artifact dependency order (cycle-safe topo ordering). - validates selected artifact dependency order (cycle-safe topo ordering).
- resolves required/optional inputs per artifact source definition. - consumes only the concrete dependency graph finalized by configuration;
planning, prerequisite closure, execution, reuse, invalidation, and semantic
evidence do not infer a second graph from inputs.
- resolves required/optional inputs per artifact source definition into an
ordered semantic identity. Each identity records the configured input name,
canonical source ID, required policy, explicit presence, source contract,
checksum, size, and a source-based logical identity. Workspace paths and
producer run IDs are excluded.
- orders input identities by configured input name independently of Go map
iteration. Runtime adapter paths remain a separate execution-only map.
- omits an unavailable optional input from the adapter request while retaining
explicit absence in its semantic identity; an unavailable required input
fails.
- resolves prepared stable input sources through the shared manifest-authoritative
identity resolver; it does not accept incidental files or fall back to
campaign/session source paths.
- reuses checksums and sizes from validated prepared, extraction, and current
configured-artifact evidence. Other resolved inputs are hashed as confined
regular files with streaming reads and the central resolved-artifact size
limit.
- owns a versioned SHA-256 fingerprint contract with one fixed-field canonical
JSON payload and no map serialization. Configured artifacts are fingerprinted
in deterministic dependency order.
- fingerprints the normalized artifact key, prompt and profile identifiers,
effective render-debug behavior, session-relative output identity, sorted
dependency keys, ordered input declarations and semantic identities,
validated current dependency-output identities, and sorted effective
Scriptorium variables (including Narratio's sticky session variable).
- provides read-only reconciliation that classifies each configured record as
current, stale, missing, failed, or otherwise non-resumable, and
separately identifies manifest records removed from current configuration.
A record is current only when its fingerprint version and value match and its
configured output still passes manifest-authoritative evidence validation.
- owns a read-only typed work planner. Its explicit targets are enabled
artifacts by default or the exact normalized `--artifacts` selection when
supplied. It closes targets over configured prerequisites, orders the closure
topologically, reuses current members, and schedules every non-current member
before its dependents.
- force applies only to explicit targets. A current prerequisite is reused
unless it is itself an explicit forced target; disabled prerequisites may be
rebuilt when required, while unrelated disabled artifacts are excluded.
- the work plan carries explicit targets, prerequisite-only work, deterministic
execution and reuse lists, invalidated and removed records, and a cloned
reconciled starting collection. Valid unrelated configured records survive
reconciliation, removed records are omitted, and incidental files never
become current without manifest-backed regeneration.
- implements aggregate resume validation by running the same read-only catalog,
fingerprint reconciliation, and work planner used by execution. A succeeded
aggregate record is reusable exactly when the selected closure schedules no
artifact work and no removed configuration records require a checkpoint;
stale unrelated records do not block a partial selection.
- exposes the typed artifact decision to `session plan`. Planning applies it to
a cloned manifest after modeling earlier selected stage transitions, so
aggregate run/skip and artifact execute/reuse decisions match the ordinary
runner without creating durable state or invoking Scriptorium.
- executes only the work plan's scheduled entries. Manifest-validated current
prerequisites remain available through the runtime catalog without invoking
Scriptorium; newly produced prerequisites enter that catalog with the same
contract, checksum, and size identity used for persisted current evidence.
- before the first adapter call, checkpoints the reconciled starting collection
when it removed configuration records or cleared obsolete authority. After
each successful artifact, checkpoints the complete collection before making
that artifact available to the next scheduled dependent.
- keeps adapter output in the invocation's run-local analyze directory until
it is a safe, non-empty, bounded regular file with a calculated checksum and
complete output contract. Canonical replacement uses the shared atomic file
operation boundary and verifies that the installed checksum matches the
validated run-local bytes.
- records each successful artifact's freshly computed fingerprint, canonical
relative output path, contract, checksum, size, producer run ID, bounded
Scriptorium provenance, logs, and generated configuration references in the
analyze-owned manifest collection.
- preserves valid unrelated current records during partial execution. If a
rebuilt output's bytes and contract are unchanged, unselected dependents may
remain current. If that semantic identity changes, unselected transitive
dependents become stale without being executed; dependents included in the
invocation are evaluated in dependency order instead.
- returns aggregate outputs from every current session record in normalized-key
order after the complete reconciled collection has been checkpointed;
unrelated stale records do not make an otherwise successful partial
invocation fail.
- resolves previous-session sources from local `previous/` cache only. - resolves previous-session sources from local `previous/` cache only.
- runs optional render-debug, then artifact execution. - runs optional render-debug, then artifact execution.
- validates non-empty output files and materializes canonical outputs. - validates non-empty output files and materializes canonical outputs.
## Failure Semantics ## Failure Semantics
- required missing configured/previous-session inputs fail. - required missing configured/previous-session inputs fail.
- missing required prepared stable input source includes prepare rerun guidance.
- missing required previous-session source includes prepare rerun guidance. - missing required previous-session source includes prepare rerun guidance.
- missing required `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` inputs includes render rerun guidance. - missing required `narratio.transcript.final_markdown` or
`narratio.transcript.final_trimmed_markdown` inputs includes render rerun
guidance.
- dependency cycles or unavailable required dependencies fail. - dependency cycles or unavailable required dependencies fail.
- adapter validation failures fail stage. - adapter validation failures fail stage.
- a scheduled artifact failure installs a retry collection on the live manifest
with the active artifact marked `failed`, a bounded error, and no output
authority. Current transitive dependents become stale without execution.
- earlier artifacts remain current only after their run-local output passed
validation, canonical materialization, and a successful complete-collection
manifest checkpoint; unattempted later artifacts retain their reconciled
session state.
- unrelated current records survive a partial failure. Old canonical bytes for
the failed artifact and newly materialized bytes whose checkpoint fails are
incidental, not current evidence.
- a checkpoint failure stops the queue immediately. The in-memory retry
collection starts from the last successful checkpoint, marks the active
artifact failed without output authority, and stales its current transitive
dependents.
- the runner's ordinary error path marks aggregate analyze failed, clears its
aggregate outputs, invalidates publish, notify, and cleanup through the
application dependency relation, and persists the live retry collection.
Terminal-save errors are joined with the original artifact or checkpoint
failure.
## Invariants ## Invariants
- `analyze` performs no remote storage calls for previous-session source resolution. - `analyze` performs no remote storage calls for previous-session source resolution.
- input-identity resolution is read-only: it does not invoke adapters,
materialize outputs, update status, or create run-local files.
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
IDs, executable and config paths, workspace roots, diagnostic locations, and
executable or private transitive configuration contents. A change that is
visible only inside Scriptorium—such as a file privately loaded by its config
path—requires an explicit forced regeneration.
- output provenance and metadata are deterministic per execution. - output provenance and metadata are deterministic per execution.
- a canonical file without current per-artifact manifest evidence is never
promoted to current state.
- an ordinary retry after interruption or partial failure reuses every valid
earlier checkpoint and schedules only failed, stale, missing, or otherwise
non-current work in dependency order.
## Related Contracts And Tests
- [Configuration](../config.md#scriptorium-artifact-entries) owns artifact
fields and source-selection rules, including
[artifact families](../config.md#scriptorium-artifact-families).
- [CLI](../cli.md) owns user-visible artifact selection.
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
- Implementation and tests: `internal/stage/analyze.go`,
`internal/stage/analyze_artifact_execution.go`,
`internal/stage/analyze_execution_state.go`,
`internal/stage/analyze_input_identity.go`,
`internal/stage/analyze_fingerprint.go`,
`internal/stage/analyze_reconciliation.go`, `internal/stage/analyze_plan.go`,
`internal/stage/analyze_resume.go`, `internal/stage/analyze_test.go`,
`internal/stage/analyze_checkpoint_test.go`,
`internal/stage/analyze_input_identity_test.go`,
`internal/stage/analyze_fingerprint_test.go`,
`internal/stage/analyze_reconciliation_test.go`,
`internal/stage/analyze_plan_test.go`,
`internal/stage/analyze_incremental_execution_test.go`,
`internal/stage/analyze_failure_test.go`,
`internal/stage/analyze_resume_test.go`, and
`internal/app/analyze_checkpoint_recovery_test.go`.

View File

@@ -0,0 +1,54 @@
# Stage: cleanup
## Purpose
Remove explicitly configured local data for the run selected by a successful
publish commit. Cleanup is the twelfth and final canonical stage, after
`notify`.
## Authorization And Targets
Cleanup is disabled unless at least one of these policies is true:
- `pipeline.spool.delete_audio_after_publish`
- `pipeline.workspace.cleanup_after_publish`
An enabled cleanup requires the manifest's publish stage to be succeeded with
`uploaded=true`, a canonical `published_run_id`, and the exact remote commit and
current commit-pointer keys for the manifest campaign/session prefix. The
configured campaign and session must match the manifest. Cleanup derives its
only possible targets through the canonical artifact path helpers:
- the published run's session spool-audio directory; and
- the published run's workspace run directory.
It does not scan parent directories, infer the newest run, or read object
storage. Every deletion uses the root-confined file operation, which rejects a
root itself, traversal, symlink escape, and out-of-root targets.
## Execution And Resume
Disabled cleanup records an explicit skipped result. Enabled cleanup removes
targets in deterministic policy order and treats an already-missing target as
success. A successful record contains the published run ID and completed policy
names.
A previous cleanup success is reusable only while its semantic configuration
matches and every selected target remains absent. A recreated target makes the
stage non-resumable. A failure is recorded against cleanup; publish remains
succeeded and remote current state is unchanged. On a later normal full run,
valid upstream stages are skipped and cleanup is retried without republishing.
`narratio publish` selects publish only and never invokes cleanup. A bounded run
through `notify` or any earlier endpoint also excludes it. Operators may run
`narratio run-stage cleanup <session_id>` directly; missing, failed, malformed,
or identity-mismatched publication evidence fails before deletion.
## Implementation And Tests
- Planning and evidence validation: `internal/stage/cleanup_plan.go`
- Execution and resume: `internal/stage/cleanup.go`
- Root-confined deletion: `internal/fileops/cleanup.go`
- Focused tests: `internal/stage/cleanup_plan_test.go`,
`internal/stage/cleanup_test.go`, and
`internal/app/cleanup_orchestration_test.go`

View File

@@ -0,0 +1,148 @@
# Internal: Extract Stage
## Responsibility
`extract` runs after `render` and before `analyze`. It converts the canonical
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
An omitted or disabled Notarius section makes the stage explicitly self-skip
with reason `notarius_disabled`, no outputs, and no Notarius runner.
The external protocol is documented in the
[Notarius integration contract](../integrations/notarius.md). Configuration
fields belong in [Configuration](../config.md), and physical paths and force
procedures belong in [Operations](../operations.md).
## Lifecycle
`internal/stage/extract.go`:
1. resolves the final trimmed transcript from the shared artifact catalog;
2. resolves every configured prepared reference through the shared
manifest-authoritative identity resolver before creating run-local output;
3. resolves each reference to a stable content-addressed session snapshot,
reusing verified bytes or reconstructing a missing/corrupt regular entry,
and rejects any source change observed while copying;
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
including sorted reference identities and the explicit downstream
configuration-profile selection;
5. creates a run-local staging directory and invokes the injected
`notarius.Runner`, carrying the optional already-resolved credential
reference without exposing its value, the optional exact configuration
profile, plus effective resume and chunk-cache controls after force
precedence;
6. revalidates the reference snapshots, then validates the v2 successful
receipt, confined index, management documents, configured required lane
descriptors, validation summaries, and regular payload files;
7. atomically promotes the complete bundle to its immutable durable location;
8. records one non-selectable `notarius_index` output and one selectable
`notarius_lane` output per configured lane; and
9. registers each lane as `narratio.extraction.<output_key>` for downstream
Scriptorium and publish resolution.
Lane records retain checksum, contract, producer run ID, and Notarius system,
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
root, receipt, diagnostic paths, rejection/warning summaries, producing
Narratio run ID, the resolved trimmed-input identity, and invocation
fingerprint. The input identity binds the exact transcript bytes, canonical
source ID, producer stage/output/run identity, and resolution provenance.
Reference metadata contains only selector, source ID, canonical session-relative
path, checksum, and size; adapter requests receive selector and absolute stable
snapshot path, never payload contents. Snapshot bytes must
match the prepared identity both before and after Notarius runs, so a concurrent
prepared-file replacement cannot make recorded provenance describe different
bytes from those supplied to Notarius.
Validation completes before the single durable rename, so a rejected result
cannot expose a partial durable bundle.
Snapshot destinations are owned by `internal/artifacts` under
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
or timestamp and can be reused across invocations. Identical bytes and basename
share a verified file; changed bytes select a different digest directory.
Unsafe path components and non-regular conflicts fail without recursive
deletion. Snapshot paths are delivery details and never become manifest output
evidence or prerequisites for reuse of an already complete extract bundle.
Any executed extraction outcome that replaces a different effective outcome
marks succeeded analysis and delivery dependents stale. Render is an independent
sibling and remains current. Repeating the same disabled self-skip with no
outputs is stable and does not repeatedly invalidate dependent stages.
## Resume Validation
Before the focused validator runs, the application compares extract's versioned
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
reference selector/source mappings, sorted declared output contracts, and each
canonical `narratio.extraction.<key>` output identity. It also covers the
explicit Notarius configuration-profile selection; omitted, empty, and exact
`default` selectors share the base-selection identity. It excludes executable,
timeout, working directory, config path, and private Notarius config contents.
Credential presence, reference name, and resolved value are likewise
operational-only and excluded. Ordinary command loading still resolves a
selected reference before resume evaluation, but a credential-only change does
not rerun a valid completed extraction or stale its dependents.
`internal/stage/extract_resume.go` then permits a skip only when the existing
stage record still matches the current byte- and provenance-bearing invocation
evidence. That evidence covers the current direct trimmed-transcript identity,
sorted prepared-reference identities, pipeline identity, and configured output
contracts, plus the same downstream profile-selection identity. The same
reference helper and transcript identity are resolved again for artifact
evidence, so changing current transcript bytes, reference bytes, producer
identity, or a named downstream selection makes the prior extraction obsolete.
Operational runner settings do not invalidate otherwise current durable
evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
make a complete extraction non-resumable. If extraction executes, ordinary
runs forward the configured values, while force disables resume and selects
`refresh`.
A valid prepared-reference change makes extraction non-resumable. Missing,
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
with prepare-force guidance because an immediate extract rerun cannot succeed.
The validator then checks the producing run identity, canonical immutable
bundle root, path confinement and absence of symlink components, receipt
identity, exactly one canonical index, the exact configured source set,
contracts and provenance, regular-file status, and stored checksums. Missing or
obsolete results are non-resumable and run again; unsafe filesystem conditions
return an error rather than silently accepting or replacing data.
This sensitivity is limited to the explicit downstream selector. Narratio's
own profile label and whole effective-configuration digest are not blanket
extraction invalidators. Neither contract can observe files imported by
Notarius configuration, profile contents, prompt/module definitions, or other
transitive inputs. Operators must force extraction after changing any such
private input behind a stable identifier.
## Failure Behavior
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
index compatibility, inconsistent warning or diagnostic envelopes,
required-lane rejection or incomplete validation, payload inspection,
checksum, or installation errors fail the stage through ordinary manifest
transition handling.
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
not given selectable extraction sources unless the complete configured result
has passed validation and installation.
When a replacement attempt begins, the current session-stage record no longer
advertises payload from the previous success. A failed replacement therefore
has no current outputs, logs, generated configuration references, or metadata,
while the earlier immutable promoted bundle remains available for audit and
recovery.
## Implementation And Focused Tests
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
`internal/stage/extract_resume.go`,
`internal/stage/extract_test.go`,
`internal/stage/semantic_contracts_delivery.go`
- Stable snapshot materialization and verification:
`internal/stage/extract_references.go`
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
`internal/adapters/notarius/subprocess_test.go`
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
`internal/artifacts/extraction_catalog_test.go`
- Composition and downstream behavior: `internal/app/runner_test.go`,
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
- Opt-in Notarius v0.7.0 production-boundary coverage:
`internal/stage/notarius_cache_integration_test.go`

View File

@@ -1,25 +1,57 @@
# Stage: merge # Stage: merge
## Purpose ## Purpose
Normalize raw transcript inputs and merge into base transcript via Seriatim. Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Inputs ## Inputs
- `transcripts/raw/*.json` - `transcripts/raw/*.json`
- `inputs/speakers.yml` - `inputs/speakers.yml`
- `inputs/autocorrect.yml` - `inputs/autocorrect.yml`
## Outputs ## Outputs
- `transcripts/base.json` - `transcripts/base.json`
- optional `artifacts/seriatim.report.json` - optional `artifacts/seriatim.report.json`
## Key Behavior ## Key Behavior
- discovers and validates raw transcript inputs.
- discovers and validates raw transcript inputs. For versioned transcribe
state, discovery requires aggregate transcribe success and consumes only the
aggregate outputs that exactly match the current per-speaker collection.
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
workflows without versioned speaker state retain directory discovery.
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output. - normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
- merges normalized inputs (`seriatim.Run`) into base transcript. - merges normalized inputs (`seriatim.Run`) into base transcript.
- validates merged transcript and optional report JSON. - validates merged transcript and optional report JSON.
- materializes canonical outputs and records stage logs/generated configs. - materializes canonical outputs and records stage logs/generated configs.
## Invariants ## Invariants
- merge always consumes normalized forms of raw inputs. - merge always consumes normalized forms of raw inputs.
- a versioned transcribe collection is usable only through its complete,
successful aggregate output declaration.
- base transcript must validate before stage success. - base transcript must validate before stage success.
- report output is config-gated. - report output is config-gated.
## Resume Evidence
Merge records a versioned semantic-configuration fingerprint for the Seriatim
merge operation, output schema, coalesce gap, and every configured advanced
merge transformation. A change reruns merge and stales only its fixed
descendants; prepare and transcribe remain reusable. Binary path, timeout,
report emission, logs, and diagnostic retention are operational exclusions.
Configuration or resources loaded privately inside Seriatim are outside
Narratio's observable contract and require `--force` when changed. An existing
successful merge record without evidence reruns once when selected.
## Related Contracts And Tests
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
- [Configuration](../config.md#pipeline) owns operator-selected Seriatim values.
- Implementation and tests: `internal/stage/merge.go`,
`internal/stage/merge_test.go`,
`internal/stage/semantic_contracts_initial.go`, and
`internal/stage/semantic_contracts_initial_test.go`

View File

@@ -1,22 +1,42 @@
# Stage: normalize # Stage: normalize
## Purpose ## Purpose
Normalize polished transcript into final transcript using Seriatim. Normalize polished transcript into final transcript using Seriatim.
## Inputs ## Inputs
- `transcripts/polished.json` - `transcripts/polished.json`
## Outputs ## Outputs
- `transcripts/final.json` (or configured normalize output path) - `transcripts/final.json` (or configured normalize output path)
- optional `artifacts/seriatim.normalize.report.json` - optional `artifacts/seriatim.normalize.report.json`
## Key Behavior ## Key Behavior
- resolves polished transcript from manifest outputs/canonical fallback.
- resolves polished transcript only from verified manifest output evidence.
- applies `pipeline.normalize` config or default normalize config. - applies `pipeline.normalize` config or default normalize config.
- runs Seriatim normalize with configured timeout/binary. - runs Seriatim normalize with configured timeout/binary.
- validates normalized transcript and optional report. - validates normalized transcript and optional report.
- materializes canonical outputs and records logs/generated configs. - materializes canonical outputs and records logs/generated configs.
## Invariants ## Invariants
- final transcript must validate as processed transcript JSON (`segments` array). - final transcript must validate as processed transcript JSON (`segments` array).
- normalize defaults are applied when `pipeline.normalize` is unset. - normalize defaults are applied when `pipeline.normalize` is unset.
## Resume Semantics
The versioned semantic fingerprint covers the Seriatim normalize operation,
output schema and canonical output identity, plus the configured transcript
transformations. Seriatim's executable and timeout and optional report
generation are operational and do not invalidate the normalized transcript.
## Related Contracts And Tests
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
- [Configuration](../config.md#pipeline) owns normalize fields and defaults.
- Implementation and tests: `internal/stage/normalize.go`,
`internal/stage/normalize_test.go`,
`internal/stage/semantic_contracts_refinement.go`

View File

@@ -1,23 +1,54 @@
# Stage: polish # Stage: polish
## Purpose ## Purpose
Run Audita polishing on base transcript and produce polished transcript. Run Audita polishing on base transcript and produce polished transcript.
## Inputs ## Inputs
- `transcripts/base.json` - `transcripts/base.json`
- `inputs/glossary.yml` - `inputs/glossary.yml`
## Outputs ## Outputs
- `transcripts/polished.json` - `transcripts/polished.json`
- optional `artifacts/audita.report.json` - optional `artifacts/audita.report.json`
## Key Behavior ## Key Behavior
- resolves base transcript from merge outputs/canonical fallback.
- invokes Audita with configured model/module/runtime options. - resolves base transcript only from verified merge manifest output evidence.
- invokes an Audita runner configured with static model/runtime options; the
invocation supplies paths and modules.
- validates processed transcript structure (`segments` array required). - validates processed transcript structure (`segments` array required).
- validates optional report JSON. - validates optional report JSON.
- materializes canonical outputs; records logs/generated config and adapter metadata. - materializes canonical outputs; records logs/generated config and adapter metadata.
## Invariants ## Invariants
- polished transcript schema validation is mandatory. - polished transcript schema validation is mandatory.
- report output is config-gated. - report output is config-gated.
## Resume Semantics
The versioned semantic fingerprint covers the Audita service endpoint, model,
validation model, module set, transcript description, output schema, selected
external configuration path, and canonical polished-transcript identity. Module
ordering is normalized because the configured modules form a set. Audita's
executable, timeouts, concurrency, report and debug behavior, work retention,
and resolved credential value are operational and do not invalidate a
successful result. The public credential reference name may participate in
configuration provenance but never exposes its resolved value.
Narratio can fingerprint a selected model, module, or configuration identifier,
but it cannot inspect content that Audita privately resolves behind that stable
identifier. Force `polish` after changing such private content without changing
its identifier.
## Related Contracts And Tests
- [Audita](../integrations/audita.md) owns subprocess, validation, and failure
semantics.
- [Configuration](../config.md#pipeline) owns operator-selected Audita values.
- Implementation and tests: `internal/stage/polish.go`,
`internal/stage/polish_test.go`,
`internal/stage/semantic_contracts_refinement.go`

View File

@@ -1,42 +1,99 @@
# Stage: prepare # Stage: prepare
## Purpose ## Purpose
Materialize canonical current-session inputs before processing stages. Materialize canonical current-session inputs before processing stages.
## Inputs ## Inputs
- resolved `campaign.yml`, `session.yml`, and pipeline config
- stable input files (`speakers`, `autocorrect`, `glossary`) - resolved campaign, session, and pipeline configuration
- audio source: - stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
- local `audio_dir`/`audio_files`, or - optional spell-catalog overlay
- S3 `audio_s3.prefix` - one resolved local or S3 audio source
- enabled configured artifact input requirements for previous-session sources - enabled configured artifact input requirements for previous-session sources
## Outputs ## Outputs
- `inputs/campaign.yml` - `inputs/campaign.yml`
- `inputs/session.yml` - `inputs/session.yml`
- `inputs/pipeline.resolved.yml` - `inputs/pipeline.resolved.yml`
- `inputs/speakers.yml` - `inputs/speakers.yml`
- `inputs/autocorrect.yml` - `inputs/autocorrect.yml`
- `inputs/glossary.yml` - `inputs/glossary.yml`
- `inputs/players.yml`
- `inputs/party.yml`
- optional `inputs/spell_catalog.json`
- `audio/*.flac` - `audio/*.flac`
- optional `previous/manifest.json` - optional `previous/manifest.json`
- optional `previous/artifacts/**` - optional `previous/artifacts/**`
- deterministic `manifest.inputs` entries (checksums + provenance) - deterministic `manifest.inputs` entries (checksums + provenance)
## Key Behavior ## Key Behavior
- validates required config/store state. - validates required config/store state.
- enforces local audio vs S3 audio mutual exclusivity. - enforces local audio vs S3 audio mutual exclusivity.
- rejects duplicate explicit local audio sources after resolution.
- gives distinct local source paths with the same basename deterministic unique
prepared filenames so neither source is overwritten.
- materializes S3 audio through spool/cache-aware logic. - materializes S3 audio through spool/cache-aware logic.
- materializes a configured spell catalog with checksum and provenance, or
safely removes an obsolete canonical spell catalog and its manifest record
when the effective input is omitted.
- copies the validated canonical party bytes unchanged and deterministically
generates the prepared players projection.
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements. - scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
- when previous requirements exist: - clears managed `previous/` state on every invocation, then, when requirements exist:
- clears managed `previous/` state; - resolves the pointer-selected previous source through the shared resolver;
- builds previous-cache remote plan;
- downloads previous manifest/artifacts; - downloads previous manifest/artifacts;
- records previous inputs in `manifest.inputs`. - records previous inputs in `manifest.inputs`.
Required previous-session inputs fail when unavailable; optional missing inputs are skipped. Required previous-session inputs fail when unavailable; optional missing inputs
are typed skipped results. Each requirement resolves only from its unique
`published_output` declaration in the pointer-selected remote commit. Prepare
strips that selected run's immutable `published/` prefix to derive the local
cache path, then validates the committed checksum, size, and generation.
## Invariants ## Invariants
- only `prepare` hydrates canonical `previous/` cache state. - only `prepare` hydrates canonical `previous/` cache state.
- managed previous artifacts are stored under `previous/artifacts/**` without duplicate `artifacts/artifacts/` nesting. - managed previous artifacts are stored under `previous/artifacts/**` without
duplicate `artifacts/artifacts/` nesting.
- managed `previous/` state represents only the current requirement set.
- `manifest.inputs` ordering is deterministic (`kind`, `path`). - `manifest.inputs` ordering is deterministic (`kind`, `path`).
## Resume Evidence
Prepare records a versioned semantic-configuration fingerprint for the
resolved campaign/session selection, local-versus-S3 audio mode and canonical
audio names, stable-input ownership/presence, previous-session identity, and
the party mode plus canonical players projection version, and the effective
previous-artifact requirement set. A change reruns prepare and
stales its fixed descendants. Existing successful records without this
evidence rerun once when selected.
Workspace, spool, and cache placement and absolute source relocation are not
semantic when logical selection, canonical names, and bytes are equivalent.
The fingerprint deliberately does not read or rehash large audio. Prepared
input checksums remain the content provenance. Before reusing success, prepare
validates every durable prepared copy and compares current stable-input bytes,
canonical party and derived-player bytes, local audio membership/checksums, or
S3 key/size/entity-tag identity with that provenance. Source relocation with
equivalent names and bytes remains reusable; changed or unavailable evidence
causes a normal prepare rerun.
## Related Contracts And Tests
- [Configuration](../config.md) owns audio selection, stable input fields, and
previous-session settings.
- [Operations](../operations.md) owns physical input, audio, spool, cache, and
previous-state layout.
- [Storage Internals](storage.md) and [Artifact Internals](artifacts.md) explain
the internal collaborators.
- Implementation and tests: `internal/stage/prepare.go`,
`internal/stage/prepare_test.go`,
`internal/stage/prepare_resume.go`,
`internal/stage/prepare_resume_test.go`,
`internal/stage/semantic_contracts_initial.go`,
`internal/stage/semantic_contracts_initial_test.go`,
`internal/audio/s3_audio_test.go`,
`internal/previouscache/*_test.go`

View File

@@ -1,44 +1,147 @@
# Stage: publish # Stage: publish
## Purpose ## Purpose
Upload run/session outputs to object storage and atomically advance remote current state. Upload run/session outputs to object storage and atomically advance remote current state.
## Inputs ## Inputs
- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`
- run root `runs/{run_id}/**` - successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
- publish output rules (`pipeline.publish.outputs`) - declared run-local diagnostic files
- effective publish locks (static + remote merged lock set) - resolved publish output rules
- local `previous/**` files when present - independent per-source remote lock objects read at planning start
- the validated prior current commit when an initially locked source may carry
- durable previous-session cache files when present
## Outputs ## Outputs
- uploaded run files under remote `runs/{run_id}/...` (excluding `audio/**`)
- uploaded selected publish outputs under session prefix - uploaded session manifest, declared run-local diagnostics, and selected
- uploaded `previous/**` files under session prefix when present publish outputs;
- uploaded `current/manifest.json` - uploaded durable previous-session cache files when present;
- uploaded `current/run_id.txt` written last - immutable run-scoped commit manifest; and
- current commit pointer, written last.
Exact remote placement and the operator workflow belong in
[Operations](../operations.md#publish-workflow).
## Key Behavior ## Key Behavior
- stage can self-skip when publish disabled or run upload disabled.
`publish.go` retains the transaction-owning `Run` path. Source discovery,
manifest-derived archive metadata, confined opening, and checksum validation
live in `publish_sources.go`; artifact-catalog matching, selection, required
and optional rules, destination resolution, and collision inputs live in
`publish_outputs.go`. Copying, hashing, immutable upload, reuse checks, and
remote verification live in `publish_transfer.go`; committed-manifest,
immutable-commit, and current-pointer encoding and writes live in
`publish_commit.go`. `publish_output_plan.go` owns the deterministic, I/O-free
five-outcome model and its secret-free metadata projection.
- when publishing or run upload is disabled, completes successfully with no
outputs and records explanatory metadata. This is not an explicit self-skip:
the session manifest records success. Enablement and upload policy are
fingerprinted, so changing either automatically makes the prior result
non-resumable.
- validates prerequisite stage success and object-store availability. - validates prerequisite stage success and object-store availability.
- collects deterministic run file list plus run `manifest.json`. - derives a deterministic run-archive allowlist from the session manifest's
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution. declared current-run outputs, logs, and generated configurations. References
outside the current run root, including canonical session files and prior-run
diagnostics, are not archive candidates. No distinguished diagnostic file is
read or uploaded.
- opens each archive candidate beneath its archive root without following
symlinked ancestors or leaf entries, verifies that it is a regular file and
checks a declared checksum when present, then streams the opened descriptor.
- derives the durable previous-cache archive from its validated manifest using
the same confinement and regular-file checks.
- resolves publish output sources through runtime artifact catalog and
manifest-aware resolution. Configured Scriptorium outputs are publishable
only from validated `current` per-artifact analyze evidence; an incidental
canonical file, aggregate output, stale/failed/unselected record, or
mismatched path, size, or checksum remains unavailable. Built-in and
extraction sources likewise require their own current manifest evidence.
- publishes extraction lanes only through explicit configured output rules;
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
- selected artifact filter applies to configured artifact sources only. - selected artifact filter applies to configured artifact sources only.
- locked outputs are skipped intentionally (including required ones). - plans every configured output into exactly one source-sorted outcome:
- optional missing outputs are skipped; required missing unlocked outputs fail. `published`, `carried_locked`, `locked_unpublished`, `optional_missing`, or
- writes remote current manifest before current run pointer. `unselected`.
- an initially locked selected output carries its exact prior
`published_output` declaration when one exists. The declaration may point to
an earlier run in the same session and retains its source, key, checksum,
size, and generation; the bytes are not copied.
- verifies a carried object's exact key, size, and generation through object
metadata without downloading or opening the local output source.
- an initially locked source with no prior declaration remains absent even
when the output rule is required. Optional missing unlocked outputs remain
absent; required missing unlocked outputs fail.
- creates one complete immutable source-to-destination mapping before upload;
- uploads and verifies every declared immutable object and the commit manifest;
- writes the immutable commit from the combined new and exact carried
declarations.
- rereads every initially unlocked participating source after the immutable
commit. A newly present lock aborts selection, leaving the uploaded immutable
attempt unselected. Removing an initial lock may conservatively produce one
carry because the initial snapshot controls the plan.
- updates `current/commit-pointer.json` exactly once, last, and writes no other
mutable publication control record.
## Metadata Signals ## Metadata Signals
Includes counts/lists for: Includes counts/lists for:
- run uploads - run uploads
- published output uploads - published output uploads
- previous uploads - previous uploads
- skipped optional outputs - one `publish_output_outcomes` collection containing all five output states,
- skipped unselected outputs exact declared keys, and key-derived origin runs where applicable
- locked outputs - remote commit and current-pointer key paths
- current-state key paths - the run identifier selected by the commit
- `current_pointer_written`
## Invariants ## Invariants
- `current/run_id.txt` is the remote commit marker and is written last.
- run upload excludes `audio/**`. - `current/commit-pointer.json` is the remote commit marker and is written last.
- publish locks are not overridden by `--force`. - run files, selected outputs, previous-cache files, and the committed session
manifest are all declared by an immutable commit under the run prefix.
- run and previous uploads contain only manifest-declared regular files opened
from verified descriptors; symlinks, special files, replacement races, and
undeclared entries are rejected or ignored before uploads begin.
- run-local diagnostics, including Notarius receipt and stderr files, are
archived only when recorded by the session manifest. A run with no declared
current-run diagnostics legitimately uploads none.
- independent publish locks are not overridden by `--force`; every initially
unlocked relevant source is reread immediately before current-state
selection.
- successful uploaded publication records the exact run ID, remote commit key,
and current-pointer key consumed by the later cleanup stage.
## Resume Semantics
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
normalized source/destination/required output rules, and the remote backend,
bucket, region, endpoint, and root-prefix identity. Rule ordering is
canonicalized. Credential
environment names, path-addressing transport mode, local workspace placement,
and run identifiers are excluded. Independent remote locks remain mutable state
and are read directly by publish at planning and commit boundaries; semantic
evidence does not replace those safety checks.
The commit boundary and cleanup authorization are normative architecture invariants; see
[Architecture](../policy/architecture.md#publish-commit-boundary).
Publish never deletes local state. The standalone `publish` command selects
only this stage; a full run selects notify and cleanup afterward.
## Related Contracts And Tests
- [Configuration](../config.md#publish-configuration-summary) owns output
rules.
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
physical remote state.
- [Artifact Internals](artifacts.md) explains source resolution and current-state
helpers.
- Implementation and tests: `internal/stage/publish.go`,
`internal/stage/publish_sources.go`, `internal/stage/publish_outputs.go`,
`internal/stage/publish_output_plan.go`,
`internal/stage/publish_transfer.go`, `internal/stage/publish_commit.go`,
`internal/stage/publish_test.go`,
`internal/stage/semantic_contracts_delivery.go`,
`internal/app/operator_helpers_test.go`, and
`internal/app/cleanup_orchestration_test.go`

View File

@@ -1,29 +1,60 @@
# Stage: render # Stage: render
## Purpose ## Purpose
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim. Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
It runs after `trim` and before `extract` in the canonical sequence. Render and
extract are independent sibling consumers: replacing render output does not
invalidate extraction, but it does invalidate succeeded analysis and delivery
records that may consume rendered transcripts.
## Inputs ## Inputs
- `narratio.transcript.final` (`transcripts/final.json`) - `narratio.transcript.final` (`transcripts/final.json`)
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`) - `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
## Outputs ## Outputs
- `narratio.transcript.final_markdown` -> `transcripts/final.md` - `narratio.transcript.final_markdown` -> `transcripts/final.md`
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` - `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
## Key Behavior ## Key Behavior
- uses `pipeline.render` settings (enabled/format/title/booleans). - uses `pipeline.render` settings (enabled/format/title/booleans).
- resolves inputs manifest-first, then canonical fallback. - resolves inputs only from verified manifest output evidence.
- writes run-local outputs first, then materializes canonical session outputs. - writes run-local outputs first, then materializes canonical session outputs.
- records input provenance, output paths, adapter metadata, logs, and generated config refs. - records input provenance, output paths, adapter metadata, logs, and generated config refs.
- skips with stage metadata when `pipeline.render.enabled=false`. - when `pipeline.render.enabled=false`, completes successfully with no outputs
and records explanatory metadata. This is not an explicit self-skip: both
manifests record success. Because enablement is fingerprinted, enabling
render later automatically makes the prior result non-resumable.
## Failure Semantics ## Failure Semantics
- missing normalized input fails with normalize rerun guidance. - missing normalized input fails with normalize rerun guidance.
- missing trimmed input fails with trim rerun guidance. - missing trimmed input fails with trim rerun guidance.
- adapter/subprocess failure fails stage. - adapter/subprocess failure fails stage.
- empty render output files fail validation. - empty render output files fail validation.
## Invariants ## Invariants
- only `format: markdown` is supported. - only `format: markdown` is supported.
- render stage owns production of built-in Markdown transcript sources. - render stage owns production of built-in Markdown transcript sources.
## Resume Semantics
The versioned semantic fingerprint covers enablement, final format, resolved
title (including the session-title fallback), timestamp, segment-ID and
metadata inclusion, both canonical input identities, and both Markdown output
identities. Seriatim's executable, timeout, and report behavior are operational
and do not invalidate rendered transcripts. A render-only change leaves the
independent `extract` sibling reusable while invalidating their shared
downstream consumers.
## Related Contracts And Tests
- [Seriatim](../integrations/seriatim.md) owns render subprocess behavior.
- [Configuration](../config.md#pipeline) owns render fields and defaults.
- Implementation and tests: `internal/stage/render.go`,
`internal/stage/render_test.go`,
`internal/stage/semantic_contracts_refinement.go`

View File

@@ -1,22 +1,93 @@
# Stage: transcribe # Stage: transcribe
## Purpose ## Purpose
Generate raw per-speaker transcripts from prepared audio using WhisperX. Generate raw per-speaker transcripts from prepared audio using WhisperX.
## Inputs ## Inputs
- `audio/*.flac` from `prepare` - `audio/*.flac` from `prepare`
## Outputs ## Outputs
- `transcripts/raw/<speaker>.json` - `transcripts/raw/<speaker>.json`
## Key Behavior ## Key Behavior
- discovers prepared audio from manifest inputs or canonical audio directory.
- derives speaker ID from `.flac` basename. - discovers prepared audio from manifest inputs or the canonical audio
- runs WhisperX with configured concurrency/retry settings. directory and derives each speaker identity from its `.flac` filename.
- validates each output as JSON. - hashes the current audio and reconciles it with the versioned per-speaker
- writes run-local outputs then materializes canonical transcript outputs. completion collection before starting HTTP work.
- reuses only records whose audio identity, observable WhisperX semantics,
canonical output location, JSON, checksum, and size remain current.
- durably checkpoints the reconciled collection before dispatch, then sends
only missing or obsolete speakers through the bounded worker group.
- validates each successful run-local response, rechecks the audio bytes,
materializes the canonical raw transcript, and checkpoints that speaker
before accepting another completion.
- rebuilds aggregate stage outputs in stable speaker order from the complete
current collection, including records produced by earlier invocations.
## Invariants ## Invariants
- speaker basenames must be unique.
- prepared audio identities must be unique; prepare disambiguates distinct
source paths that share a basename.
- output path returned by adapter must match requested output path. - output path returned by adapter must match requested output path.
- each successful output is validated before stage success. - an empty adapter result path means the requested path; adapters cannot select
an alternate destination.
- candidate collection construction, manifest save, and installation into live
state are serialized. Expensive hashing and HTTP work remain outside that
critical section.
- a canonical file has no completion authority until its manifest checkpoint
succeeds. A save failure preserves the last successfully committed
collection for ordinary retry.
- each successful output is validated before its checkpoint, and cancellation,
incomplete dispatch, or a failed aggregate save cannot be reported as
aggregate stage success.
- removed audio inputs are omitted from the current collection and aggregate
output list; reconciliation does not delete arbitrary transcript files.
- merge cannot consume a partial versioned collection: it requires transcribe
aggregate success and an exact match between current speaker records and
declared aggregate outputs.
## Resume Evidence
Each completed speaker record binds the speaker ID, session-relative audio
path, streamed audio checksum and size, the Narratio-visible WhisperX service
URL and recognition language, and the canonical transcript path, checksum,
size, and producer run. Changed audio bytes select that speaker for replacement;
a changed URL or language selects every speaker. Either case stales fixed
descendants through the normal runner decision. Retry count/delay, concurrency,
timeout, credentials, and diagnostic locations are operational and do not
change this evidence.
An ordinary retry reuses every valid speaker checkpoint even when aggregate
transcribe status is failed, running, or stale. A succeeded stage is skipped only
when its stored speaker set and aggregate output evidence also match the current
collection. Otherwise, execution repairs those records without retranscribing
reusable speakers and stales dependents through the normal runner path.
`--force` first durably replaces
the collection with an empty versioned collection, so a later ordinary retry
cannot resurrect pre-force results; speakers checkpointed by that forced
attempt remain reusable.
WhisperX models or private service configuration not exposed by Narratio's
adapter contract cannot be fingerprinted; use `--force` after changing them.
An older aggregate-only transcribe record has no trustworthy audio-to-output
binding. When transcribe is next selected, every current speaker is therefore
generated once and recorded in the versioned collection; later invocations can
reuse it normally.
## Related Contracts And Tests
- [WhisperX](../integrations/whisperx.md) owns HTTP, retry, timeout, and
cancellation semantics.
- [Configuration](../config.md#pipeline) owns concurrency and other
operator-selected values.
- Implementation and tests: `internal/stage/transcribe.go`,
`internal/stage/transcribe_reconciliation.go`,
`internal/stage/transcribe_execution_state.go`,
`internal/stage/transcribe_resume.go`,
`internal/stage/transcribe_test.go`,
`internal/stage/transcribe_reconciliation_test.go`, and
`internal/stage/transcribe_checkpoint_test.go`

View File

@@ -1,18 +1,20 @@
# Stage: trim # Stage: trim
## Purpose ## Purpose
Produce a final-trimmed transcript; optionally generate bounds-driven trim.
Produce a final-trimmed transcript. By default, the stage generates bounds and
applies a bounds-driven trim.
## Inputs ## Inputs
- `transcripts/final.json` - `transcripts/final.json`
## Outputs ## Outputs
- `transcripts/final.trimmed.json` (or configured trim output path) - `transcripts/final.trimmed.json` (or configured trim output path)
- when trim enabled: `artifacts/session_bounds.json` - when trim enabled: `artifacts/session_bounds.json`
## Key Behavior ## Key Behavior
When `trim.enabled=false`:
- copies normalized transcript to trimmed output.
When `trim.enabled=true`: When `trim.enabled=true`:
- runs Scriptorium bounds artifact generation; - runs Scriptorium bounds artifact generation;
@@ -22,7 +24,34 @@ When `trim.enabled=true`:
- either copies unchanged transcript or runs Seriatim trim; - either copies unchanged transcript or runs Seriatim trim;
- validates trimmed transcript and materializes bounds output. - validates trimmed transcript and materializes bounds output.
When `trim.enabled=false`:
- copies normalized transcript to trimmed output.
## Invariants ## Invariants
- normalized transcript is required input. - normalized transcript is required input.
- bounds output exists only in enabled trim path. - bounds output exists only in enabled trim path.
- render-debug output is diagnostic and not a declared stage output. - render-debug output is diagnostic and not a declared stage output.
## Resume Semantics
The versioned semantic fingerprint covers enablement, the bounds prompt and
profile identifiers, the Scriptorium configuration identity, transcript input
name, sticky session variable, bounds and trimmed output identities, and the
Seriatim trim operation. Diagnostic bounds rendering, diagnostic output paths,
timeouts, executable paths, and optional reports are operational and do not
invalidate the canonical trimmed transcript.
Narratio cannot inspect prompt, profile, or configuration content that
Scriptorium or Seriatim privately resolves behind a stable identifier. Force
`trim` after changing such private content without changing its identifier.
## Related Contracts And Tests
- [Scriptorium](../integrations/scriptorium.md) owns bounds generation and
debug-render subprocess behavior.
- [Seriatim](../integrations/seriatim.md) owns transcript trimming behavior.
- [Configuration](../config.md#pipeline) owns trim fields and defaults.
- Implementation and tests: `internal/stage/trim.go`,
`internal/stage/trim_test.go`,
`internal/stage/semantic_contracts_refinement.go`

View File

@@ -1,34 +1,72 @@
# Internal: Storage # Internal: Storage
## Purpose ## Purpose
Document remote object-store contracts and S3 implementation behavior.
Explain the object-store interface and S3 implementation used by Narratio.
Remote key layout and lifecycle belong in [Operations](../operations.md), while
operator-selected storage fields and credential mechanisms belong in
[Configuration](../config.md).
## Primary Contract ## Primary Contract
`storage.ObjectStore` interface: `storage.ObjectStore` interface:
- `List(ctx, prefix)` - `List(ctx, prefix)`
- `Read(ctx, key)` returns an object body and the generation observed with it
- `Download(ctx, key, localPath)` - `Download(ctx, key, localPath)`
- `Upload(ctx, localPath, key, opts)` - `Upload(ctx, localPath, key, opts)`
- `UploadConditional(ctx, source, key, opts, condition)`
- `Delete(ctx, key)` removes one object and succeeds when it is already absent
- `Exists(ctx, key)` - `Exists(ctx, key)`
Key invariant: Key invariant:
- callers pass full bucket-relative keys; - callers pass full bucket-relative keys;
- storage implementations do not infer campaign/session/run prefixes. - storage implementations do not infer campaign/session/run prefixes.
## Configuration `ReadObjectBounded` is the shared mechanism for small control objects. It opens
`NewObjectStoreFromConfig` currently supports S3-backed stores from `pipeline.storage.*` config. one object version, returns the metadata observed with that body, rejects an
oversized known size before transfer, and still performs a context-aware
limit-plus-one read. It closes the body on every exit. Callers own the policy
limit and add the control-object category to errors; this helper is not used for
large artifact payloads.
S3 constructor behavior: ## Composition
- requires configured bucket;
- uses region/endpoint/path-style options when set; `NewObjectStoreFromConfig` constructs the S3-backed implementation from
- resolves credentials from configured env var names (with defaults). resolved configuration. The application loads configured filesystem secrets
before calling it. The storage adapter consumes already-resolved values; it does
not own discovery, defaults, or configuration validation.
## S3 Backend Behavior ## S3 Backend Behavior
- normalizes object keys. - normalizes object keys.
- `List` paginates and returns normalized `ObjectInfo`. - `List` paginates and returns normalized `ObjectInfo`.
- A truncated S3 listing must supply a new, non-empty continuation token;
otherwise listing fails with bucket and prefix context instead of looping.
- `Download` writes local files with parent directory creation. - `Download` writes local files with parent directory creation.
- `Upload` streams local file and returns remote metadata. - `Upload` streams local file and returns remote metadata.
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
match or absence precondition directly to the provider request and reports a
failed precondition without performing a local check-then-write replacement.
- `Delete` maps one normalized key to the provider delete-object operation and
treats an absent object as success.
- `Exists` maps not-found responses to `false`. - `Exists` maps not-found responses to `false`.
## Invariants ## Invariants
- storage layer is stateless regarding manifest/stage progression. - storage layer is stateless regarding manifest/stage progression.
- bounded reads never retain more than the caller's limit plus one byte and do
not replace owner-specific size policy.
- publish ordering semantics are owned by stage/app code, not storage adapters. - publish ordering semantics are owned by stage/app code, not storage adapters.
- A caller that installs remote content into a canonical workspace path first
writes a sibling temporary file, validates the complete object against its
owner-specific contract, and only then replaces the destination. Temporary
files used solely for remote comparisons are removed by their callers and do
not represent installed workspace state.
## Implementation And Tests
- Contract and S3 adapter: `internal/adapters/storage`
- Composition: `internal/app/object_store.go`
- Tests: `internal/adapters/storage/*_test.go`,
`internal/app/object_store_test.go`

View File

@@ -1,57 +1,103 @@
# Internal: Workspace # Internal: Workspace
## Purpose ## Purpose
Define local session layout, run-local stage layout, and cleanup guardrails.
## Canonical Session Layout Explain the helpers that construct local session and run paths, coordinate
Session root: single-writer access, and confine cleanup. The authoritative physical layout and
- `{workspace.root}/work/{campaign}/{session_id}` retention workflow belong in [Operations](../operations.md#local-state-layout).
Core directories/files: ## Path Ownership
- `inputs/`
- `audio/`
- `transcripts/`
- `artifacts/`
- `reports/`
- `logs/`
- `config/`
- `current/`
- `runs/`
- `previous/`
- `manifest.json`
- `.lock`
`previous/` reserved files: `internal/artifacts` owns canonical session, run, spool, cache, and
- `previous/manifest.json` previous-cache path construction. `SessionPathsFor` provides the session-scoped
- `previous/artifacts/**` path model, and layout creation goes through `EnsureLayoutFor`. Callers should
consume those helpers instead of rebuilding relative paths.
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
explicit cleanup roots and removes only normalized proper descendants through
the resulting root handle.
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
`EnsureWorkspaceDirectory` reapplies the directory mode after creation so a
restrictive umask cannot remove group access, while retaining existing ownership
and group. Credential paths are outside this contract; the platform-specific
operational requirements are in [Operations](../operations.md#workspace-permissions).
## Run-Local Stage Layout ## Run-Local Stage Layout
When run context is available, stages use:
- `runs/{run_id}/{stage}/outputs/`
- `runs/{run_id}/{stage}/logs/`
- `runs/{run_id}/{stage}/reports/`
- `runs/{run_id}/{stage}/config/`
- `runs/{run_id}/{stage}/scratch/`
Run-local outputs are materialized back into canonical session paths before stage success. `internal/stage/run_local.go` maps stage outputs and diagnostics into an
`previous/**` writes are never redirected to run-local output paths. invocation-scoped layout. Successful outputs are validated and atomically
materialized into canonical session paths before stage success. Managed
previous-session cache paths remain session-durable and are never redirected
into run-local output space.
Extraction uses run-local receipt, stderr, and output-root helpers, then
validates and renames the external bundle to the unique immutable Notarius
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
[Operations](../operations.md#extraction-workflow).
Prepared references delivered to Notarius use session-durable paths owned by
`internal/artifacts` beneath
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
content-addressed snapshots are outside run-local layout, survive ordinary
published-run cleanup, and are excluded from publish and restore enumeration.
## Locking ## Locking
`artifacts.LocalStore` enforces single-writer session lock via `.lock` file (`ErrLockConflict` on contention).
`artifacts.LocalStore` enforces the single-writer session lock via an
operating-system lock held on `.lock` (`ErrLockConflict` on contention). The
file retains owner metadata after release or process death; its existence is
not evidence that a lock is active. Command and restore flows wait for this
lock only while their context remains active, and report a release failure.
The runner assigns every invocation a run-workspace identity but creates the
physical directory only before an executing stage that requires it. Fully
reused invocations and cleanup-only invocations therefore do not accumulate
empty run directories.
## Cleanup Semantics ## Cleanup Semantics
Automatic post-publish cleanup:
- only runs when publish actually executed and succeeded;
- requires `uploaded=true` and `current_pointer_written=true` metadata;
- respects `pipeline.spool.delete_audio_after_publish` and `pipeline.workspace.cleanup_after_publish`;
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
Manual clean command: The canonical cleanup stage:
- `clean <session_id>` removes session work and spool subtree.
- `clean --all` removes all workspace work and spool children. - runs after notify only when selected by the invocation range;
- durable cache is preserved unless `--clear-cache` is requested. - requires a succeeded publish record with `uploaded=true`, the published run
ID, remote commit key, and current commit-pointer key;
- consumes the resolved cleanup policy described in
[Configuration](../config.md);
- refuses unsafe deletes (root delete, traversal, and out-of-root targets);
- treats missing targets as successful, idempotent deletion; and
- uses its ordinary failed/succeeded manifest stage record for retry. A later
full run can skip reusable upstream stages and retry cleanup without
republishing.
Manual cleanup uses the same root-confined deletion mechanism. Invocation
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
[Operations](../operations.md#cleanup).
## Invariants ## Invariants
- campaign-aware session root is mandatory. - campaign-aware session root is mandatory.
- manifest-driven stage state is durable across runs. - manifest-driven stage state is durable across runs.
- cleanup guardrails prevent destructive root/out-of-scope deletion. - cleanup guardrails prevent destructive root/out-of-scope deletion.
- restore installation is confined to validated relative paths beneath the
selected session root.
- ordinary workspace paths retain group-writable directory and file modes across
nested creation, replacement, and Notarius promotion.
## Implementation And Tests
- Path model and local store: `internal/artifacts/paths.go`,
`internal/artifacts/local.go`
- Run-local materialization: `internal/stage/run_local.go`
- Immutable bundle installation: `internal/stage/extract.go`
- Workspace modes: `internal/fileops/modes.go`
- Cleanup confinement: `internal/fileops/cleanup.go`,
`internal/stage/cleanup.go`, `internal/stage/cleanup_plan.go`, and
`internal/app/cleanup_targets.go`
- Tests: `internal/artifacts/paths_model_test.go`,
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
`internal/fileops/modes_posix_test.go`,
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
`internal/stage/cleanup_test.go`, and
`internal/app/cleanup_orchestration_test.go`

View File

@@ -36,6 +36,13 @@ narratio session init 2026-04-04 --remote --force
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values. If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
Campaigns must provide stable input files for speakers, autocorrect, glossary,
players, and party, and may provide an optional spell-catalog overlay. Session
files may override those paths for one session. The `prepare` stage
materializes them under `inputs/`; configured consumers use the prepared files,
never the original campaign or session source paths. Field definitions and
source IDs are in [Configuration](./config.md#notarius-reference-bindings).
## Standard Session Workflow ## Standard Session Workflow
1. Select pipeline/campaign/session config. 1. Select pipeline/campaign/session config.
@@ -63,6 +70,22 @@ narratio run 2026-04-04
narratio session status 2026-04-04 narratio session status 2026-04-04
``` ```
Run, plan, and status output identify the resolved pipeline profile (or `none`)
and effective configuration digest. Status distinguishes the current resolved
value from the last value persisted in the session manifest, which helps
diagnose profile switches without changing resume authority.
Before switching an operational profile, compare its effective meaning with the
production base through `narratio config diff default <profile>`.
The command is read-only and succeeds whether it finds differences or not. Its
sorted records describe defaulted, expanded concrete configuration—not source
file layout—so it can be used to review model, artifact, and publish changes
without creating a session or run. Use `narratio config artifacts` to review the
concrete artifact identities, definitions, outputs, and dependencies. Select the
same campaign explicitly when profiles could resolve different campaign paths;
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
for syntax and record format.
## Stage Execution and Continuation Behavior ## Stage Execution and Continuation Behavior
Canonical stage order: Canonical stage order:
@@ -74,15 +97,52 @@ Canonical stage order:
5. `normalize` 5. `normalize`
6. `trim` 6. `trim`
7. `render` 7. `render`
8. `analyze` 8. `extract`
9. `publish` 9. `analyze`
10. `notify` 10. `publish`
11. `notify`
12. `cleanup`
Execution rules: Execution rules:
- succeeded stages are skipped unless `--force` is set; - succeeded stages are skipped unless `--force` is set; stages with semantic
configuration contracts additionally require matching versioned evidence;
- `run` continues interrupted or partially completed sessions by running non-succeeded stages; - `run` continues interrupted or partially completed sessions by running non-succeeded stages;
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`. - forcing a stage marks succeeded transitive dependents as `stale` before the
replacement runs; render and extract are independent siblings; and
- an executed failure, changed self-skip, or success that replaces a different
effective outcome uses the same fixed dependency relation. A
repeated self-skip with the same reason and no outputs is stable and does not
perpetually rerun dependent work.
Every aggregate stage except analyze currently provides semantic-configuration
evidence; transcribe additionally reconciles per-speaker completion evidence,
while analyze retains its more precise per-artifact fingerprints and validator.
Prepare additionally validates current stable/local/S3 source identity and the
checksums of its durable prepared copies before reuse. Changed bytes, audio
membership, S3 object identity, or missing/tampered copies rerun prepare and
its fixed descendants without requiring `--force`. Changing prepare selection
semantics likewise reruns all fixed descendants; changing
WhisperX language/service identity reuses prepare; changing a Seriatim merge
transformation reuses prepare and transcribe; and changing an Audita model
reuses prepare, transcribe, and merge while rebuilding transcript refinement.
A trim change invalidates both render and extract through the fixed dependency
relation, while a render-only change preserves the extract sibling.
Operational timeouts, retry/concurrency tuning, executable paths,
workspace/cache/spool placement, reports, diagnostics, and secret values are
excluded. Configuration, models, prompts, modules, or resources loaded
privately inside external tools remain unobservable to Narratio. If their
contents change behind the same configured identifier, explicitly force the
affected stage.
An explicit self-skip is a durable `skipped` stage outcome that later runs
reconsider. It differs from successful no-output execution: disabled `render`
and `publish`, and absent or no-executable `analyze`, record `succeeded` with
metadata and no outputs. Ordinary later runs reuse those successful results;
force the affected stage after enabling or configuring it. Optional artifact
inputs are omitted only from the consuming artifact invocation and do not make
the stage self-skip.
Single-stage execution: Single-stage execution:
@@ -90,16 +150,322 @@ Single-stage execution:
narratio run-stage normalize 2026-04-04 --force narratio run-stage normalize 2026-04-04 --force
``` ```
Contiguous bounded execution uses inclusive canonical endpoints:
```bash
narratio session plan 2026-04-04 --from extract --through analyze --force
narratio run 2026-04-04 --from extract --through analyze --force
```
Omitting `--from` selects from `prepare`; omitting `--through` selects through
`cleanup`. Force applies only within the selected range. Repeating `--from`,
`--through`, or `--force` is rejected instead of resolving by argument order.
The plan command uses the same selection contract and prints only the selected
range. Planning is read-only: it clones the loaded manifest, models selected
stage transitions and invalidation in memory, and invokes resume validation
without writing the manifest, creating run directories, materializing files,
or invoking pipeline adapters. Analyze detail separates explicit targets,
prerequisite rebuilds, scheduled execution, and current reuse. This lets a
coarsely stale aggregate analyze stage show zero artifact executions when its
selected artifact evidence is still semantically current.
Before a bounded run or plan whose range starts after `prepare`, every excluded
prefix stage must already have a session-manifest status of `succeeded` or
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
or interrupted prerequisite without creating a run-local directory or changing
session state. Widen `--from` to include that stage, or recover it explicitly
before retrying. Excluded prefix stages are not resume-validated or repaired as
part of the bounded invocation; selected stages still reject missing, unsafe,
or manifest-inconsistent inputs at their owning boundary.
Stages after `--through` are not prerequisites and are never scheduled by the
bounded invocation. A selected forced stage can mark one of those succeeded
dependents stale through the fixed invalidation relation, but the dependent
does not execute until a later invocation selects it. Production composition
likewise initializes only collaborators needed by the selected range and
shared session lifecycle. In particular, render does not require Notarius or
Scriptorium, extract does not require Scriptorium, and analyze does not require
the transcription, Seriatim, Audita, or Notarius adapters.
### Transcription continuation and replacement
Transcribe checkpoints each validated speaker independently. If a later
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
save fails, the aggregate stage is not successful and downstream merge cannot
consume the partial set. The last successfully saved speaker records and their
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
After correcting the underlying problem, retry normally:
```bash
narratio run-stage transcribe 2026-04-04
```
Narratio hashes the current prepared audio, validates recorded transcript JSON,
checksum, size, and location, and calls WhisperX only for missing or obsolete
speakers. A changed audio file replaces only its speaker; a changed WhisperX
URL or language makes every current speaker non-reusable. Retry, timeout, and
concurrency tuning do not invalidate completed speakers.
Use force only when every current transcript must be replaced, including after
a WhisperX model or private server-side configuration changes behind the same
URL and language:
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Force durably clears the reusable collection before sending requests. If that
forced attempt fails, a later ordinary retry reuses only replacement speakers
checkpointed after the reset; it does not revive pre-force results. Do not edit
speaker checksums or copy older JSON into the canonical directory to influence
reuse.
The first current Narratio invocation that selects transcribe for an older
aggregate-only manifest retranscribes every current speaker once because the
old record cannot prove which audio bytes produced each file. Subsequent runs
reuse the new versioned evidence normally. The overall session-manifest schema
identifier is unchanged, but older strict-decoding Narratio binaries may reject
a manifest after current Narratio writes the additive speaker fields; retain a
backup or complete the upgrade before relying on downgrade.
Restoring a manifest does not implicitly publish or restore every raw speaker
file. If a recorded canonical transcript is absent locally, ordinary
reconciliation schedules that speaker again. Removing run directories does not
remove canonical checkpointed transcripts and does not by itself invalidate
them.
## Artifact Development Workflows
Choose the narrowest workflow that matches the intended replacement:
| Intent | Command | Preserved and rebuilt state |
| --- | --- | --- |
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
All three stop before publication. Inspect the results and run
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
For the extraction-plus-analysis development loop, use:
```bash
narratio regenerate-artifacts 2026-04-04
narratio regenerate-artifacts 2026-04-04 --artifacts session_recap,player_handout
```
This command is a transparent expansion to a forced bounded `run` from
`extract` through `analyze`. Extraction always rebuilds its complete configured
bundle. Analysis rebuilds the selected targets and their required analysis
prerequisites, or uses the normal default selection when no artifact names are
given. The command does not run publish, notify, or cleanup; delivery and
retention remain separate operator actions.
Inspect current artifact evidence, then publish explicitly when the regenerated
set is ready:
```bash
narratio session artifacts 2026-04-04
narratio publish 2026-04-04
```
If planning or execution reports stale, missing, failed, or tampered
analysis evidence, regenerate the affected target instead of copying an older
canonical file into place or editing the manifest. See
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
Analyze checkpoints each completed configured artifact before starting later
artifact work. After a partial failure or interruption, retry normally: valid
earlier checkpoints are reused and execution resumes with failed, stale, or
missing work in dependency order. `--force` is unnecessary unless the operator
intends to replace otherwise-current artifacts.
### Developing With A Notarius Configuration Profile
A Narratio development overlay can explicitly select a Notarius overlay without
requiring the names to match. For example, `profiles/testing.yml` may contain:
```yaml
notarius:
config_profile: downstream-economy
```
Selecting Narratio's `testing` profile then supplies `downstream-economy` only
when extraction executes. Switching that explicit selector makes a recorded
extraction non-current and reruns it when `extract` is in the selected stage
range; the usual dependent invalidation then applies. Selecting a profile does
not create a separate workspace or retain a separate current extraction result.
Narratio cannot observe edits inside Notarius configuration behind an unchanged
selector. Force extraction after such an edit when a replacement is intended.
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
use `analyze` when the existing extraction bundle should be preserved. Field
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
and the exact downstream invocation belongs in the
[Notarius integration contract](./integrations/notarius.md#invocation).
## Artifact Selection ## Artifact Selection
`--artifacts` can be used on `run`, `run-stage`, `analyze`, and `publish`. For a configured artifact family, selecting its family key expands to every
concrete character artifact. Select a concrete generated key to operate on one
member only. Manifests and plan output retain the concrete key as the durable
identity and include the family and character ID as optional provenance.
`--artifacts` can be used on `run`, `session plan`, `run-stage`, `analyze`, and
`publish`. For a bounded run or plan, the selected range must contain `analyze`
or `publish`.
Selection behavior: Selection behavior:
- validates names against `pipeline.scriptorium.artifacts`; - validates names against `pipeline.scriptorium.artifacts`;
- filters analyze execution to selected configured artifacts; - selects explicit analyze targets and permits their required configured
prerequisites to be reused or rebuilt first;
- filters publish rules for `narratio.artifact.<name>` sources only; - filters publish rules for `narratio.artifact.<name>` sources only;
- does not suppress built-in transcript or bounds publish sources. - does not suppress built-in transcript, bounds, or explicitly configured
`narratio.extraction.<name>` publish sources; and
- never partially selects Notarius lanes.
## Extraction Workflow
When Notarius is omitted or disabled, `extract` records an explicit skipped
outcome with reason `notarius_disabled` and no outputs. A later invocation
reconsiders the skipped stage, so enabling Notarius does not require force.
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
and preserves the complete validated Notarius bundle at:
- `artifacts/notarius/{narratio_run_id}/`
The directory is immutable once promoted. Configured lanes become
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
the bundle and `index.json` are retained for audit and resume validation but
are not selectable or published implicitly.
Configured Notarius references resolve only from the current manifest-backed
prepared inputs. Their canonical locations are `inputs/party.yml`,
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
content-addressed copies under
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
SHA-256 digest identifies the reference bytes, so retries and later invocations
reuse the same path while changed bytes select a new path. Older snapshots are
retained and are not current artifact evidence. A concurrent refresh of
canonical prepared files therefore cannot change the bytes consumed by an
in-flight invocation.
For a canonical party, preparation retains the validated authored party bytes
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
The manifest records their checksums separately, with the players input marked
as derived from the party; refresh preparation after changing the roster rather
than editing either prepared file.
Inspect the effective stable-input inventory and
prepared-file readiness with:
```bash
narratio session status 2026-04-04
narratio session validate 2026-04-04
```
Reference metadata records selector, source ID, session-relative path,
checksum, and byte size, but never payload contents. Changing a prepared
reference changes extraction identity: ordinary continuation rejects the old
result, reruns Notarius, and marks successful downstream stages stale. If the
prepared file is missing or inconsistent with its manifest checksum, repair
the source configuration and refresh prepared state first:
```bash
narratio run-stage prepare 2026-04-04 --force
```
Starting a replacement clears the previous extraction payload from the current
session-stage record. If that replacement fails or self-skips, the current
record does not fall back to the earlier outputs. The earlier immutable bundle
remains available for inspection, but downstream resolution requires a new
current successful extraction record.
Notarius extraction validates its run-local bundle and moves it once to the
unique durable bundle path. A pre-existing durable destination is a collision;
no duplicate promotion tree is created.
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
inside an executed Notarius invocation. Ordinary extraction forwards their
configured values. Forced extraction always disables checkpoint loading and
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
Refresh regenerates and replaces only the active input's accepted chunk plan;
it is not cache deletion.
Notarius checkpoint recording must be enabled in the deployed Notarius
configuration. Narratio neither creates a Notarius pipeline/configuration nor
reads it to infer checkpoint availability. Configure explicit persistent
Notarius chunk-plan and checkpoint roots when reuse must survive separate
Narratio subprocesses. A changed reference, transcript, selected profile,
profile content, endpoint, or other Notarius checkpoint identity input can
produce a legitimate cold miss.
There are two independent resume boundaries: Narratio may skip a completed
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
reuse compatible accepted work only after Narratio has decided extraction must
execute. Deleting Narratio's local reference snapshots does not invalidate a
complete durable extraction; missing snapshots are reconstructed only when a
later extraction actually runs.
## External Command Lifecycle
When an external command is cancelled or times out, Narratio terminates its
owned descendants as well as the command itself. Cancellation first requests
termination where the platform supports it, then force terminates after a
bounded wait. A command is not considered finished until its leader has been
reaped, and descendants that keep standard output or error open cannot keep
the invocation blocked. Other operating systems fail closed rather than launch
a command without tree ownership.
Subprocess stdout and stderr diagnostics are separately redacted and capped at
8 MiB per invocation. Narratio does not retain configured credential values in
these logs or their error tails; reaching a capture limit terminates the command
tree and reports which stream exceeded the limit.
Run-local diagnostics are:
- `runs/{run_id}/extract/notarius.receipt.json`
- `runs/{run_id}/extract/notarius.stderr.log`
- `runs/{run_id}/extract/notarius-output/` before durable promotion
The run upload is an allowlist derived from the session manifest's declared
current-run outputs, logs, and generated configurations, not a workspace scan.
References to canonical session files and prior-run paths are excluded. Each
declared source is opened without following symlinked ancestors or the leaf,
verified as a regular file, and streamed from that verified descriptor.
Unlisted files and unsafe entries are never uploaded; a run with no declared
current-run diagnostics uploads no run-local files. No distinguished diagnostic
file is uploaded. The durable bundle is never scanned for implicit publication;
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
To intentionally replace the current extraction result, run:
```bash
narratio run-stage extract 2026-04-04 --force
```
Narratio automatically reruns extraction when its recorded invocation contract,
prepared Narratio reference identities, or durable output validation changes.
The semantic portion covers Notarius enablement, pipeline identity, declared
reference mapping, output contracts, and the explicit Notarius configuration
profile selector. Executable, timeout, working directory, and private
config-file paths are operational and do not invalidate a current result.
It cannot fingerprint configuration or profile contents, prompts, modules, or
other references loaded transitively by Notarius itself. Force extraction after
changing any of those inputs behind an unchanged selector, even when the
top-level Narratio and Notarius config paths remain the same. A forced extract
marks successful downstream stages stale. Ordinary extraction failures or
outcome changes also stale affected downstream stages, while an identical
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
Publish reuse additionally tracks enabled/run-upload behavior, normalized
output rules, and remote backend/bucket/region/endpoint/root identity.
Credential environment names, local workspace placement, and run IDs are
excluded. Regardless of semantic reuse evidence, executing publish reads
independent remote locks directly and rechecks initially unlocked sources
before commit selection.
## Publish Workflow ## Publish Workflow
@@ -115,29 +481,70 @@ Equivalent:
narratio run-stage publish 2026-04-04 --force narratio run-stage publish 2026-04-04 --force
``` ```
This standalone command stops after publish and does not invoke cleanup. A
default full `run` continues through notify and cleanup.
Publish commit model: Publish commit model:
- uploads run files under `{session_prefix}/runs/{run_id}/`; - uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
- uploads configured published outputs; audio and the run-local Notarius staging bundle;
- uploads `previous/**` cache files when present; - uploads configured published outputs and `previous/**` cache files into the
- writes `current/manifest.json`; same immutable run scope, including only explicitly configured extraction
- writes `current/run_id.txt` last. lanes;
- writes `{session_prefix}/runs/{run_id}/commit.json` after all declared
immutable objects are uploaded and verified; and
- writes `{session_prefix}/current/commit-pointer.json` once, last.
`current/run_id.txt` is the remote current-state commit marker. `current/commit-pointer.json` is the remote current-state commit marker. It
selects exactly one immutable commit, which declares the complete object set.
## Remote Commit Migration
The immutable remote commit contract uses
`runs/{run_id}/commit.json` to declare a run's complete object set and a small
`current/commit-pointer.json` to select it. The pointer binds the selected
commit by version, checksum, size, and storage generation; committed artifacts
are also checksum- and generation-bound. Readers accept this contract now and
strictly reject mismatched or unknown data.
A selected commit may carry a `published_output` declaration from an earlier
run in the same campaign/session. Restore, previous-session caching, status,
and artifact inspection use that declaration's exact immutable key and stable
source; they do not reconstruct a current-run key or scan the session prefix.
Other artifact types remain confined to the selected commit's own run.
When the pointer is absent, remote state is not published; republish the
session to establish a committed snapshot. Remote commit state does not carry
local `current_pointer_written` metadata.
## Publish Locks ## Publish Locks
Lock sources: Each mutable lock is an independent strict JSON object at
`{session_prefix}/locks/{sha256(source)}.lock`. The lowercase digest is computed
from the exact canonical source. Listing validates every object in that
reserved prefix and reports locks sorted by source.
- static locks in `pipeline.publish.locks` Upgrades do not read, migrate, or delete the retired `{session_prefix}/locks.yml`
- mutable remote locks in `{session_prefix}/locks.yml` object. It has no effect on publication and may be removed manually after
verifying lock commands and publication behavior with the upgraded Narratio
version. No migration command is required.
Effective lock rules: Operational rules:
- static and remote locks are merged; - add uses conditional creation and reports an existing lock;
- static locks win on source collisions; - add with `--force` observes the current generation and attempts one
- locked outputs are intentional skips; generation-matched replacement. A concurrent change is reported for an
- lock add/remove commands mutate only remote lock state. explicit operator retry;
- remove is idempotent and reports whether a lock was present;
- the initial publish lock snapshot controls output planning. A locked source
carries its exact declaration from the prior current commit when available;
a never-published locked source remains absent even when required;
- carried bytes are not copied. Publish verifies the prior object's exact key,
size, and generation without downloading it; and
- after immutable uploads and the immutable commit, publish re-reads every
initially unlocked participating source. A newly added lock prevents the
current pointer from advancing, although the immutable attempt may remain.
A removed initial lock may conservatively cause one carry.
Examples: Examples:
@@ -163,20 +570,39 @@ Apply:
narratio session restore 2026-04-04 narratio session restore 2026-04-04
``` ```
`--dry-run` does not write durable session files. It still reads the selected
remote current state and may read object identity/content needed to classify the
plan, so it is not a network-free operation.
Default restore scope: Default restore scope:
- `manifest.json` - the committed session manifest and the committed transcript/artifact objects
- `transcripts/**` declared by the selected remote commit
- `artifacts/**` - `previous/**` when needed by configured previous-session artifact inputs;
- `previous/**` when needed by configured previous-session artifact inputs each is selected from its exact committed `published_output` declaration
Optional: Optional:
- `--include-audio` to include `audio/**` - `--include-audio` to include `audio/**`
- `--force` to overwrite local conflicts - `--force` to overwrite eligible conflicting regular files; it never replaces
directories, symlinks, or other non-regular local targets
Restore treats a local leaf symlink or any symlinked ancestor beneath the
selected session root as a conflict, even when the linked content matches the
selected remote object. `--force` does not override that conflict.
Before each install, restore validates the destination again beneath the
selected session root. A target changed into a symlink, directory, or other
non-regular entry is not overwritten.
Restore writes an execution report at `reports/restore-latest.json`. Restore writes an execution report at `reports/restore-latest.json`.
If restore fails after beginning installation, it leaves a durable
`.restore-incomplete.json` marker in the session root. Pipeline runs will stop
until you rerun the same restore command and it completes. Restore intentionally
does not try to roll back files already installed; retrying the selected remote
snapshot is the recovery procedure.
## Local State Layout ## Local State Layout
Session root: Session root:
@@ -195,6 +621,24 @@ Durable session paths:
- `logs/**` - `logs/**`
- `config/**` - `config/**`
- `runs/**` - `runs/**`
- `cache/notarius/references/<sha256>/<prepared-basename>`
`manifest.json` is the versioned session-progress document. Narratio accepts
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
add missing markers. Move it aside and regenerate it, restore a verified
current committed snapshot, or republish healthy regenerated state as the
relevant workflow requires.
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
stderr, and pre-promotion output remain in the producing run's `extract`
directory as described in [Extraction Workflow](#extraction-workflow).
Stable reference snapshots live in the session cache path above. Ordinary
published-run cleanup removes its selected run-local target but retains this
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
or digest directory manually only while no command is active for the session;
regular missing or corrupt content is reconstructed on the next executed
extraction, while symlinks, directories, and other unsafe conflicts must be
investigated and removed deliberately.
Run-local layout: Run-local layout:
@@ -213,6 +657,50 @@ Cache layout (durable S3 audio cache):
- `{cache.root}/s3/{bucket}/...` - `{cache.root}/s3/{bucket}/...`
Each cached audio file has an adjacent managed identity record. It binds the
file to its remote object version and verified digest; deleting or altering the
record simply causes Narratio to download and verify the object again.
Remote audio and restored files are fully transferred into sibling temporary
files and validated before they replace their canonical workspace paths.
### Workspace Permissions
Ordinary Narratio workspace content is intentionally shareable with the
workspace group. On POSIX systems, Narratio-created workspace, spool, and cache
directories converge on setgid `02775`; ordinary files, including manifests,
transcripts, generated configuration, logs, reports, and Notarius artifacts,
converge on `0664`. Narratio explicitly applies these modes so a restrictive
caller umask does not remove group write or setgid. It does not change file or
directory ownership: the configured workspace's existing group is inherited.
API keys are credentials, not ordinary workspace data. Store them outside the
shared workspace or in a separately restricted credential location; ordinary
workspace group access must never be treated as authorization to read keys.
Provision a credential directory as `0700` and credential files as `0600`;
these permission modes are an operator requirement and are not enforced by
ConfigKit.
Every pipeline-loading command bootstraps credentials once, before the first
configuration load and before any adapter is constructed. `--secrets-dir`
selects an explicit directory (relative paths use caller CWD). Otherwise a
present `secrets/` beside the resolved root pipeline is used, while an absent
conventional directory is skipped. Explicit missing/empty selections and
invalid existing directories fail the command. ConfigKit follows a symlink used
as the selected directory, but immediate entries must be regular non-symlink
files. Existing environment variables—including empty values—always win.
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
environment references. When both are absent, the AWS SDK retains its default
credential provider chain. When both are configured, Narratio supplies their
load-time resolved values through an explicit static provider. Audita's optional
`llm_api_key` is likewise captured at pipeline load and supplied only through
the child process's `AUDITA_LLM_API_KEY` override.
External adapter results are individually bounded before Narratio validates or
materializes them. These per-file limits do not reserve disk space: prevent hard
disk exhaustion with filesystem, service, container, or volume quotas sized for
the session workload.
## Cleanup ## Cleanup
Session-scoped cleanup: Session-scoped cleanup:
@@ -238,9 +726,30 @@ Rules:
- `clean` deletes work/spool session state; - `clean` deletes work/spool session state;
- cache is preserved unless `--clear-cache` is set; - cache is preserved unless `--clear-cache` is set;
- automatic post-publish cleanup is gated by successful publish commit plus: - each deletion is confined beneath its configured workspace, spool, or cache
root; root, traversal, and out-of-root targets are refused;
- the final cleanup stage is gated by successful publish evidence plus:
- `pipeline.spool.delete_audio_after_publish=true` - `pipeline.spool.delete_audio_after_publish=true`
- `pipeline.workspace.cleanup_after_publish=true` - `pipeline.workspace.cleanup_after_publish=true`
- the manifest publish record has `uploaded=true`, the published run ID, the
exact remote commit key, and the current commit-pointer key.
Cleanup derives only the configured run-scoped spool and workspace targets,
uses root-confined deletion, and does not read or mutate object storage. A
failure leaves publish succeeded and the remote committed snapshot current.
Retry with a normal full run: valid upstream work, including publish and notify,
is skipped while the failed cleanup stage runs again. `run-stage cleanup` is
also available when the manifest has valid publish evidence. A range ending at
`notify` or earlier and the standalone `publish` command never delete local
state.
Older local manifests that contain the retired cleanup-obligation field are
intentionally incompatible with strict manifest decoding. If such an
obligation is still outstanding, complete it with the pre-upgrade binary.
Otherwise verify that the remote publication is committed, move aside or
regenerate the incompatible local manifest (or restore the verified remote
snapshot), and then use the cleanup stage. Manual JSON editing is not the
normal migration path.
## Operational Caveats ## Operational Caveats
@@ -248,4 +757,5 @@ Rules:
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded. - Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`). - Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
- Restore requires configured object storage and committed remote current state. - Restore requires configured object storage and committed remote current state.
- Storage-backed commands load filesystem secrets before object-store initialization. - Pipeline-loading commands bootstrap filesystem secrets once before configuration
resolution and object-store initialization.

View File

@@ -1,202 +1,305 @@
# Narratio Architecture # Architecture
## Purpose This document defines Narratio's intended high-level architecture and the
invariants that changes must preserve. Implemented component details belong in
the [Internal Overview](../internal/overview.md) and its linked documents.
Significant architectural decision history belongs under `docs/adr/` when such
records exist.
`narratio` is a Go orchestration application for processing D&D session audio into polished transcripts and generated session artifacts. ## System Shape
This document defines the development principles for the project. It is inward-facing: its audience is developers and LLM coding agents. It should guide future changes, not serve as a complete implementation reference. Narratio is a small Go application that turns D&D session audio into polished
transcripts and generated session artifacts. It is an explicit, stage-driven
orchestrator, not a general workflow engine.
Implemented component details belong under `docs/internal/`. Narratio coordinates specialized external systems rather than reimplementing
their domains:
## Project Shape - WhisperX performs transcription;
- Seriatim performs deterministic transcript processing and rendering;
- Audita performs transcript correction and polishing;
- Notarius extracts validated structured artifact bundles; and
- Scriptorium executes prompts and produces configured artifacts.
Narratio is a modular, stage-driven orchestrator. Narratio owns orchestration, configuration resolution, session and run state,
artifact and path modeling, manifest persistence, stage sequencing, resume,
restore, cleanup gates, and publish semantics. External contracts are defined
in the [integration documentation](../integrations/).
It coordinates specialized downstream systems rather than reimplementing their domains: The pipeline has one canonical ordered stage set. Configuration may enable,
disable, or parameterize supported behavior, but it must not turn that sequence
into an arbitrary DAG or hide orchestration in generic workflow abstractions.
An invocation selects either the full sequence or one inclusive contiguous
range of it. Execution remains flat and canonical even though invalidation is
dependency-aware: the application owns a separate fixed relation used only to
stale transitive dependents, including dependents outside a selected range.
The implemented stage inventory belongs in the
[Internal Overview](../internal/overview.md).
- WhisperX handles transcription. Narratio is contract-first without being abstraction-heavy. Interfaces and
- Seriatim handles deterministic transcript merge/normalization/trim behavior. extension points should protect demonstrated boundaries. New abstraction is not
- Audita handles transcript correction and polishing. itself an architectural goal.
- Scriptorium handles prompt execution and generated artifacts.
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics. ## Ownership And Dependency Direction
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine. The application boundary owns command dispatch, configuration selection,
production composition, session locking, and top-level lifecycle. It may depend
on concrete implementations to assemble a run.
## Core Principles Stage orchestration expresses intent in Narratio-level data and interfaces.
Stages may depend on configuration, manifest, artifact, path, and adapter
contracts, but they must not depend on transport-specific request types,
subprocess argument construction, cloud SDK types, or downstream tool internals.
### Modular and composable Adapters translate between Narratio contracts and external systems. They own
HTTP, subprocess, notification, and object-storage mechanics, including command
construction, transport behavior, provider response handling, and external
error adaptation. External dependency types must remain inside the adapter that
owns them unless that dependency is the adapter's explicit public contract.
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
construction, notification transport, and object-storage SDK details remain
behind these boundaries.
Code should be organized around clear responsibilities. Stages, adapters, config loading, manifest persistence, path construction, and storage behavior should remain separable and independently testable. State and path services must not infer stage policy. Storage implementations
receive explicit bucket-relative keys and do not infer campaign, session, run,
or root-prefix semantics. Manifest persistence records transitions but does not
choose orchestration policy. Artifact resolution identifies and validates
artifacts but does not execute producers.
### Hexagonal boundaries Dependencies should remain narrow and point toward Narratio-owned contracts.
Prefer the Go standard library. Add an external dependency only when it provides
a clear correctness, security, interoperability, or complexity benefit, and
confine it to the boundary that needs it.
External systems should be isolated behind narrow adapters. Stage logic should depend on Narratio-level interfaces and data structures, not on external SDK types, subprocess argument construction, or transport-specific details. ## Stage Boundaries
### Standard library preference Each stage has one explicit responsibility and declares:
Prefer the Go standard library. Add dependencies only when they provide substantial value, are necessary for an external integration, or are a widely used de facto standard.
Accepted examples include a YAML library for configuration and the AWS SDK for S3-compatible storage.
### Explicit orchestration
The pipeline should remain stage-driven and explicit. New behavior should be added through clear stage, adapter, config, or manifest contracts rather than implicit side effects or generic workflow abstraction.
## Stage Design
Each stage should have a clear scope of responsibility.
A stage should define:
- its purpose;
- required input state; - required input state;
- produced output state; - produced output state;
- config fields it consumes; - configuration it consumes;
- external adapters it uses; - external adapters it uses;
- manifest refs it reads or writes; - manifest references and metadata it reads or writes;
- skip, force, and resume behavior; - skip, force, invalidation, and resume behavior; and
- failure behavior; - failure behavior.
- tests that protect its contract.
Stages should avoid reaching across boundaries. If shared behavior is needed, prefer a helper or service with a narrow interface over duplicating ad hoc logic between stages. Stages write and validate run-local results before materializing canonical
outputs where that distinction applies. A stage is complete only after its
required outputs have been written, validated, and recorded in durable manifest
state. Later stages depend on recorded success and artifact resolution, not
merely on incidental files existing on disk.
## Transactionality and Resume A failed or interrupted stage must not be presented as successful. Failure
should preserve enough local state and diagnostics for inspection, recovery,
and resume. Forcing a stage invalidates succeeded transitive dependents
according to a fixed application-owned relation that is separate from canonical
execution order. The relation is validated against the stage inventory and is
not configurable.
A stage should behave transactionally. A stage may explicitly self-skip with a stable reason and no outputs. That
outcome is persisted, clears older outputs owned by the stage, and is
reconsidered on a later invocation. A stage may also validate whether an
otherwise successful recorded result is still resumable; an obsolete result
is staled and rerun, while an unsafe condition that prevents a sound decision
stops execution.
A stage is complete only when its outputs have been written, validated, and recorded in the manifest. If a stage fails, Narratio should preserve enough local state for inspection, recovery, and resume. Shared behavior should live behind a narrow service or helper with one clear
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
artifact, or path policy ad hoc.
A failed or incomplete run must not be treated as successful. Later stages should depend on manifest-recorded success, not merely on incidental files existing on disk. ## Manifest, Resume, And Restore
## Manifest Model The session manifest is the durable ledger for progress across invocations. It
records session and run identity, stage state, input and output references,
diagnostic references, checksums or provenance where useful, and non-secret
adapter and publish metadata.
The manifest is the durable local ledger for a run. Resume and skip decisions are manifest-driven. Filesystem state may be
inspected and validated, but file presence alone does not replace recorded
stage state. Each invocation retains an in-memory command summary and a
run-local workspace identity, but neither is a durable progress ledger. The
physical run directory is created on demand only when an executing stage
requires it; a fully reused invocation or a cleanup-only invocation does not
leave an empty run directory.
It should record: Restore treats committed remote current state as its authority. It must plan
deterministically, confine remote-to-local paths, protect local conflicts, and
install the validated session manifest after other restored durable files. The
physical workflow and recovery procedures belong in
[Operations](../operations.md).
- run identity; Restore and runner transitions for one session use the same local lock. A
- stage status; durable incomplete-restore marker blocks runner reuse after a partial restore;
- input and output refs; safe retry, rather than rollback of arbitrary local effects, is the recovery
- logs and generated config refs; mechanism. Restored manifest-local references must be confined to the selected
- checksums or provenance where useful; local session root, never trusted as producer-machine absolute paths.
- non-secret adapter and publish metadata.
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress. For the immutable remote-commit protocol, a restore or status operation binds
to one pointer-selected commit and only its declared object identities. A force
flag may replace an eligible regular managed file, but never turns a directory
or other non-regular conflict into a successful restore.
## Adapter Boundaries ## Configuration
Adapters own external integration details. Configuration is strict, explicit, centralized, and operator-oriented.
Expected boundaries: - YAML decoding rejects unknown fields.
- Defaults are centralized and testable.
- Empty configured values do not silently replace meaningful defaults.
- Validation rejects invalid configuration before stage execution where
practical.
- Root-owned imports and one selected profile resolve deterministically through
the configuration owner; commands do not implement their own merge rules.
- Canonical party rosters are campaign-owned. Their derived players projection
and concrete character-family artifacts are resolved before runtime stages
or adapters receive configuration.
- Resume uses stage- or artifact-owned semantic evidence for observable
result-affecting configuration; profile identity and an effective digest are
provenance, never blanket cache keys.
- Session templating remains narrow and deterministic rather than becoming a
general configuration language.
- Secret values are supplied indirectly and are not persisted in ordinary
configuration.
- WhisperX HTTP details stay in the WhisperX adapter. Narratio must not become a second configuration system for downstream tools.
- Seriatim CLI construction stays in the Seriatim adapter. External systems own their runtime defaults wherever practical; Narratio passes
- Audita CLI construction stays in the Audita adapter. the paths required by its stage contracts and explicit operator overrides. The
- Scriptorium CLI construction stays in the Scriptorium adapter. field-level contract and credential-supply mechanisms belong in
- Object-storage details stay behind the storage adapter interface. [Configuration](../config.md).
- AWS SDK types stay inside the S3 storage implementation.
Stage code should express intent in Narratio terms and call adapters through narrow contracts. ## Artifacts, Paths, And Storage
## Configuration Philosophy Artifact identities and local and remote paths are application contracts.
Canonical helpers own workspace, spool, cache, session, run, input, transcript,
artifact, log, report, configuration, and publish-current paths. Callers must
not reconstruct canonical paths through scattered string concatenation.
Configuration should be strict, explicit, and operator-friendly. Reusable audio cache entries require a typed record that binds a confined,
no-follow regular file and its digest to the selected remote object identity.
Size alone and unqualified multipart ETags are not content-integrity evidence.
Principles: Artifact resolution is deterministic and manifest-aware. Producers materialize
canonical outputs before reporting success, and consumers resolve declared
artifact identities rather than infer files from unrelated directory contents.
External artifact bundles become current only through validated immutable
promotion and manifest records; directory presence alone never establishes
availability.
- YAML decoding should reject unknown fields. Writes, moves, replacements, and deletions must use narrow, explicit,
- Defaults should be centralized and testable. root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
- Empty configured values should not silently override meaningful defaults. relative destinations must not expand the scope of an operation. Cleanup is
- Session templating should remain narrow and deterministic. permitted only through explicit operator action or the configured final cleanup
- Template support should serve operator convenience, not become a general configuration language. stage, and it must preserve durable cache unless cache removal is explicitly
requested.
Narratio should not become a secondary configuration system for downstream tools. Seriatim, Audita, and Scriptorium should own their runtime defaults wherever practical. Narratio should pass required stage-contract paths and explicit operator overrides. Physical layout, retention, and operational lifecycle belong in
[Operations](../operations.md). Logical external formats and durable integration
contracts belong under [Integrations](../integrations/).
## Path and Storage Discipline ## Publish Commit Boundary
Local and remote paths are part of Narratio’s application contract. Publish has one explicit remote commit boundary. A remote run becomes current
only after Narratio has successfully uploaded its immutable run-scoped objects,
the immutable commit manifest, and finally the current commit pointer.
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation. `current/commit-pointer.json` is the sole mutable selector and must be written
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
must not be presented as current remote state. Publish locks remain authoritative
and are not bypassed by a forced run. Each mutable remote lock is an independent
per-source object; lock mutations use provider-enforced generation preconditions.
Publish revalidates every initially unlocked participating source after writing
the immutable commit and before pointer selection. A lock observed then leaves
the prior committed snapshot current. A lock created after that final check may
race the pointer write; the commit protocol does not claim a stronger distributed
transaction boundary.
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics. Automatic local cleanup is permitted only after a successful publish commit,
only when explicitly configured, and only through the path-safety guardrails.
It is the final ordinary pipeline stage after `notify`. The stage derives its
exact run-scoped targets from validated publish metadata and the configured
campaign/session identity; it never infers a target from directory contents or
rereads remote storage. A cleanup failure makes the invocation incomplete while
leaving publish succeeded and the committed remote snapshot authoritative. A
normal later full run reuses valid upstream stages and retries cleanup.
## Publish Invariants The standalone `publish` command selects only publish and therefore never
invokes cleanup. Operators may select cleanup directly only when the manifest
contains valid successful publication evidence.
Publish behavior must preserve a clear commit boundary. ## Security, Privacy, And Diagnostics
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`. Narratio distinguishes ordinary workspace data from credentials. Campaign and
session material—including manifests, transcripts, prompts, generated
configuration, logs, reports, diagnostics, and Notarius artifacts—is
intentionally shareable with the configured workspace group. API-key material
is sensitive and is not covered by the ordinary workspace-sharing policy.
`current/run_id.txt` is the final remote commit marker and must be written last. Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
a cross-build gate and release asset, but no native runtime release gate.
Windows and macOS amd64 are unsupported and have no compatibility promise or
release asset.
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured. On supported systems, Narratio-created ordinary workspace directories converge on
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
umask is restrictive. This preserves the existing workspace group for nested
creation and atomic replacements without changing ownership. API-key storage
uses a separate restrictive contract.
## Security and Privacy Raw secrets must not be stored in pipeline, campaign, or session YAML or written
to manifests, logs, generated configuration, reports, publish metadata,
documentation, or examples. Pipeline configuration names environment variables;
command-owned startup may populate previously unset variables from ConfigKit's
explicit or conventional secrets directory. This bootstrap occurs once per
invocation before any typed configuration load or adapter construction and is
not hidden inside library loaders. Safe bootstrap diagnostics may identify
files and statuses but never values. Diagnostics should avoid transcript and
prompt content unless a deliberate, bounded inspection mechanism requires it.
Narratio handles private campaign material. Logs, reports, generated invocation files, generated configuration, and render
debug files are diagnostics, not canonical pipeline products. They should be
durable and discoverable where configured, and manifest references must preserve
the distinction between diagnostics and artifacts.
Rules: Documentation security rules belong in the
[Documentation Policy](documentation.md). Credential supply belongs in
[Configuration](../config.md), while permissions, sensitive runtime-artifact
handling, and recovery belong in [Operations](../operations.md).
- Do not store raw secrets in pipeline or session YAML. ## Determinism And Testability
- Use environment variable names or secret-file references for secret handling.
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
## Diagnostics Narratio prefers deterministic behavior where practical, including stable local
and remote layouts, sorted operation order, predictable generated
configuration, repeatable command construction, deterministic artifact
resolution, and reproducible planning.
Diagnostics should be durable and discoverable, but distinct from canonical outputs. Run IDs and timestamps may be intentionally variable, but surrounding behavior
must remain controllable in tests. Core behavior should be testable without live
external services; expensive, nondeterministic, destructive, or external
boundaries should be replaceable with focused test doubles. General testing
philosophy and sufficiency rules belong in the [Testing Policy](testing.md).
Logs, reports, generated invocation/config files, and render-debug files support debugging. Transcript tiers and configured artifacts are pipeline products. ## Documentation And Decision Records
Manifest refs should preserve that distinction. Documentation follows the [Documentation Policy](documentation.md). Current
behavior belongs in its canonical user, operator, integration, architecture, or
internal owner. Proposed behavior and implementation status belong under
`docs/roadmap/`.
## Determinism Significant architectural decisions may be recorded under `docs/adr/` using the
format and lifecycle defined by the documentation policy. ADR acceptance does
not establish that a decision has been implemented.
Where practical, Narratio should prefer deterministic behavior: ## Architectural Non-Goals
- stable local path layout; Narratio does not aim to provide:
- stable remote key layout;
- sorted upload order;
- predictable generated config files;
- repeatable command construction;
- tests that do not depend on live external services.
Run IDs and timestamps may be intentionally variable, but surrounding behavior should remain testable.
## Testing Expectations
Core behavior should be testable without live external services.
Tests should cover:
- config loading, defaults, and validation;
- CLI parsing and command construction;
- path helpers;
- manifest transitions;
- stage success, failure, skip, and resume behavior;
- adapter command construction;
- fake storage behavior;
- publish commit ordering;
- example config validity where practical.
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
## Documentation Expectations
Documentation must follow `docs/documentation/policy.md`.
Current behavior belongs in user-facing docs and `docs/internal/`. Future, planned, aspirational, experimental, or unimplemented work belongs only under `docs/roadmap/`.
`docs/architecture.md` should remain concise and principle-focused. It should not duplicate the full config reference, CLI reference, operations guide, or internal stage documentation.
## Non-Goals
Narratio is not:
- a generic DAG or workflow engine; - a generic DAG or workflow engine;
- a replacement configuration layer for Seriatim, Audita, or Scriptorium; - a replacement configuration layer for WhisperX, Seriatim, Audita,
- a storage backend abstraction beyond the needs of this pipeline; Scriptorium, or other downstream tools;
- a place to embed raw secrets; - a storage abstraction broader than the needs of this pipeline;
- a place for stage logic to depend directly on AWS SDK types or downstream tool internals; - stage logic coupled directly to cloud SDKs, transports, subprocess details,
or downstream implementation internals;
- raw-secret persistence;
- implicit cross-stage behavior that bypasses manifest and artifact contracts;
or
- a prompt-authoring system. - a prompt-authoring system.

View File

@@ -1,94 +0,0 @@
# Development Guide
## Purpose
Canonical contributor workflow and engineering conventions for implemented Narratio behavior.
## Repository layout
- `cmd/narratio/`: CLI entrypoint.
- `internal/app/`: command handlers, run/stage orchestration, cleanup gates, secrets loading.
- `internal/config/`: strict YAML loading, defaults, and validation.
- `internal/stage/`: stage implementations and stage registry/order.
- `internal/adapters/`: external boundary adapters (WhisperX, Seriatim, Audita, Scriptorium, storage, notify).
- `internal/manifest/`: session/run manifest types and persistence.
- `internal/artifacts/`: canonical local/remote path helpers and local artifact store.
- `docs/`: canonical documentation set.
- `examples/`: maintained config examples used by tests.
## Build and test commands
- Run focused CLI behavior checks:
```bash
go test ./internal/app -run TestExecute -v
```
- Run config example load/validate checks:
```bash
go test ./internal/config -run TestExamplesLoadAndValidate -v
```
- Run full test suite:
```bash
go test ./...
```
## Coding conventions
- Keep orchestration explicit and stage-driven; do not introduce generic workflow/DAG abstractions.
- Keep external-system details inside adapter packages; stages should consume Narratio-level contracts only.
- Use centralized path helpers from `internal/artifacts` rather than ad hoc path concatenation.
- Preserve manifest-driven state transitions (`running`, `succeeded`, `failed`, `skipped`, `stale`) as the source of run progress.
- Keep user/operator docs implementation-accurate; planned work belongs only under `docs/roadmap/`.
For design principles and invariants, see [docs/architecture.md](./architecture.md). For stage/adapter contracts, see [docs/internal/README.md](./internal/README.md).
## Dependency policy
- Prefer Go standard library where practical.
- Add third-party dependencies only when they provide clear value for required behavior.
- Keep dependency additions narrow to the boundary package that needs them.
## Change playbooks
### Add config fields
1. Add fields to config structs in `internal/config`.
2. Set defaults in `internal/config/defaults.go` when appropriate.
3. Add validation rules in `internal/config/validate.go`.
4. Add or update load/validate tests in `internal/config/*_test.go`.
5. Update canonical config docs and examples:
- [docs/config.md](./config.md)
- relevant files under `examples/`
### Add CLI flags or commands
1. Update command parsing and behavior in `internal/app`.
2. Add or update command tests (`TestExecute` and command-specific tests).
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
### Add or modify stages/adapters
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
2. Keep external transport/subprocess details in `internal/adapters`.
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
4. Add/update stage and adapter tests.
5. Update internal component contracts in `docs/internal/`.
### Update examples
1. Keep canonical examples only in `examples/`.
2. Ensure examples load and validate through runtime config paths.
3. Update `internal/config/load_validate_test.go` as needed.
4. Update links in `docs/config.md` if example filenames change.
### Update docs and roadmap
1. Keep implemented behavior in canonical docs (`README`, `docs/*.md`, `docs/internal/`).
2. Keep planned/unimplemented behavior only in `docs/roadmap/`.
3. After completing roadmap items, remove or mark them complete in `docs/roadmap/documentation.md`.
4. Run a link/path sweep before finalizing changes.

View File

@@ -1,356 +1,150 @@
# Go Project Documentation Policy # Documentation Policy
## Purpose ## Purpose
Project documentation must help four audiences: This policy assigns each documentation topic to one canonical owner. Its goal is
to keep Narratio documentation accurate, concise, discoverable, and resistant
1. users who need to run the application; to drift for users, operators, developers, integrators, and LLM coding agents.
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
## Core Rules ## Core Rules
### 1. Keep docs concise ### One Canonical Owner
Each document should cover a defined scope and only the essentials for that scope. Each authoritative fact belongs in one document. A non-owning document may give
a short, stable summary for orientation, but it must link to the canonical owner
Avoid: instead of repeating volatile details.
- long background explanations;
- repeated reference material; Volatile details include commands, flags, configuration fields and defaults,
- implementation detail in user-facing docs; stage or integration keys, schemas, file names, paths, status codes, retry
- aspirational language outside roadmap docs; behavior, and runtime guarantees. If readers could reasonably treat a statement
- verbose examples where one minimal example is clearer. as a contract, maintain it only in the owning document.
### 2. Document only implemented behavior outside roadmap files ### Current And Future Behavior
Unimplemented, planned, aspirational, experimental, or future work may be described only under: Outside `docs/roadmap/`, documentation describes implemented behavior only.
Partial features may be described only to their implemented boundary.
- `docs/roadmap/`
ADRs are the narrow exception: an ADR may record an accepted architectural
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist. decision before implementation, but acceptance must not be presented as proof
that the behavior exists. The roadmap owns implementation status and sequencing
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary. until the decision is implemented. Current architecture, user, operator,
integration, and internal documentation are updated when the behavior lands.
### 3. Use canonical homes
### Audience And Detail
Each type of information should have one canonical location.
Write for the document's stated audience and include only the detail needed for
Canonical homes: its owned topic. User and operator docs should not expose implementation detail.
Developer docs should link to user-facing and external contracts rather than
- project purpose and quickstart: `README.md` restate them.
- development principles: `docs/architecture.md`
- configuration reference: `docs/config.md` ### Examples
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md` Complete copyable files belong in `examples/`. Documentation may use the
- troubleshooting: `docs/troubleshooting.md` smallest illustrative snippet needed to explain its owned topic, but should link
- implemented internals: `docs/internal/` to maintained examples instead of embedding a second complete copy.
- future work: `docs/roadmap/`
- contributor workflow: `docs/development.md` Examples must be valid, secret-free, and tested where practical. Commands and
- copyable examples: `examples/` configuration used in documentation should match the application.
Other files should summarize briefly and link to the canonical source. ### Security And Privacy
### 4. Keep examples real Documentation and examples must not contain real credentials, private keys,
private environment dumps, sensitive source material, or private infrastructure
Examples should be valid, maintained, and free of secrets. details unless intentionally public. Document secret-handling mechanisms, not
secret values.
Where practical:
- example configs should load successfully; ## Canonical Ownership
- example commands should match real CLI syntax;
- important examples should be covered by tests. | Topic | Canonical owner | Owned content | Content owned elsewhere |
| --- | --- | --- | --- |
## Documentation Profiles | Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
All projects require: | Maintainer release procedure | `docs/release.md` | Version selection, candidate preparation and validation, guarded tag publication, release completion boundary, failure recovery, and optional asynchronous inspection. | Script implementation mechanics, current application contracts, and historical release summaries. |
| Historical release summary | `docs/releases/<tag>.md` | Immutable summary, compatibility, upgrade, and changes for one released version. | Current maintainer procedure and current application contract details. |
- `README.md` | Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
- `docs/architecture.md` | Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
Additional docs depend on the project. | CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, output conventions, and exit behavior. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, stage implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schemas, fields, defaults, environment overrides, validation rules, and user-selectable stage or integration settings. | Complete example files, CLI syntax, runtime state lifecycle, implementation details. |
### Small library | Operations | `docs/operations.md` | Runtime workflows, physical filesystem and remote-state layout, output and diagnostic handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical artifact schemas, implementation mechanics. |
| Troubleshooting | `docs/troubleshooting.md` | Symptom-driven diagnosis, likely causes, safe inspection steps and remedies, and links to relevant contracts. | CLI syntax, configuration definitions, operational procedures, integration contracts, implementation mechanics. |
Recommended: | Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
- `docs/development.md`, if contributor conventions are non-obvious | Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical artifact paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
### Simple CLI | Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
Required: | Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
- `docs/cli.md` | Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
Recommended:
- `docs/development.md` Documents that do not exist are required only when the corresponding interface
or responsibility exists. Do not create placeholder API, consumer, integration,
### Config-driven CLI or operations documents for behavior the application does not have.
Required: ## Boundary Rules
- `docs/cli.md`
- `docs/config.md` ### Orientation
Recommended: The README owns product orientation. The developer guide routes contributors.
- `examples/` Architecture owns normative structure. Internal overview owns the current
- `docs/development.md` concrete component map. These documents may link to one another but should not
maintain parallel package or behavior descriptions.
### Stateful or operator-facing application
### Commands, Configuration, Operations, And Troubleshooting
Required:
- `docs/cli.md`, if CLI-based CLI documentation answers how to invoke the application. Configuration
- `docs/config.md`, if config-driven documentation answers what settings mean. Operations answers what happens to
- `docs/operations.md` runtime state and how to operate or recover the application. Troubleshooting
starts from observable symptoms and links readers to the owning command,
Recommended: configuration, operational, or integration contract. When a workflow crosses
- `docs/troubleshooting.md` these topics, choose the document that owns the task and link to the other
- `examples/` contracts.
- `docs/development.md`
### Contracts And Implementation
### Modular, staged, service-oriented, or orchestration application
Integration and API documents define externally observable shapes and
Required: semantics. Internal documents explain how Narratio implements or consumes those
- `docs/cli.md`, if CLI-based contracts. Internal docs may name a field, file, or protocol to identify a
- `docs/config.md`, if config-driven dependency, but must link to its canonical contract for the definition.
- `docs/operations.md`
- `docs/internal/` ### Security Topics
- `docs/development.md`
This policy owns what documentation and examples may contain. Architecture owns
Recommended: application security invariants. Configuration owns credential-supply
- `docs/troubleshooting.md` mechanisms. Operations owns permissions and handling of sensitive runtime
- validated examples under `examples/` artifacts. Troubleshooting owns safe diagnostic and remediation guidance.
Internal docs own implementation mechanisms only.
## Required Documents
## Architecture Decision Records
### README.md
Use sequentially numbered ADR filenames such as
**Audience:** users, administrators, operators `0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
The README is the outward-facing project orientation page. 1. title;
2. status;
It should include, in order: 3. date;
4. context;
1. concise description; 5. decision;
2. elevator pitch; 6. alternatives considered;
3. shortest useful command or usage example; 7. consequences.
4. links to targeted docs.
Treat the decision content of an accepted ADR as immutable. When a decision
The README should be short. It is not a manual. changes, create a new ADR and update the earlier ADR's status to superseded.
Rejected architectural alternatives belong in the ADR; rejected product ideas
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.) belong in the roadmap.
### docs/architecture.md ## Maintenance
**Audience:** developers, LLM coding agents When behavior changes, update its canonical owner in the same change. If
ownership moves, remove the old definition and replace it with a link where
`docs/architecture.md` is required for every project. navigation remains useful.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed. Before completing documentation work:
It should include: - verify affected behavior and examples;
- check commands, flags, fields, defaults, schemas, and paths against their
- project shape; implementation;
- core design principles; - keep unimplemented behavior in the roadmap, subject to the ADR exception;
- package and boundary philosophy; - remove stale references and validate links;
- state/persistence philosophy, if applicable; - confirm that non-owning documents summarize and link rather than redefine;
- external integration philosophy, if applicable; - confirm that no secrets or sensitive private data were added.
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/architecture.md` describes development principles.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.

296
docs/policy/testing.md Normal file
View File

@@ -0,0 +1,296 @@
# Testing Policy
## Purpose
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
## Every test has a cost
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
A test must be:
- written and reviewed;
- understood by future maintainers and coding agents;
- executed in local and CI workflows;
- diagnosed when it fails;
- updated when legitimate behavior changes;
- maintained as fixtures, APIs, and dependencies evolve; and
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
A test is warranted only when the confidence it provides justifies these costs.
Apply this cost-benefit analysis at two levels:
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
## Default testing style
Use a **classical/Detroit-style** approach:
- Test observable behavior, resulting state, contracts, and invariants.
- Use real internal collaborators when they are fast and deterministic.
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
## Test execution requirements
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
## What deserves tests
Prioritize tests for:
1. Public and package-level contracts.
2. Domain rules and important invariants.
3. Boundary conditions and malformed input.
4. Failure handling, cancellation, retries, recovery, and partial success.
5. Serialization, schemas, compatibility, and round trips.
6. Previously observed or plausible regressions.
7. Representative integration and end-to-end workflows.
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
## Choose the right test boundary
Test through the narrowest stable boundary that expresses the behavior clearly.
This is often the package API, but it may instead be:
- a smaller pure function when dense domain logic is most clearly isolated there;
- a package-level operation when several internal collaborators jointly produce the behavior; or
- a larger integration boundary when correctness emerges from interaction with a real dependency.
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
## Test behavior, not implementation
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
Before adding or retaining a test, ask:
> What realistic defect would this test catch?
A test is suspect when its main purpose is to detect that someone:
- changed an internal constant;
- renamed or split a private helper;
- reordered equivalent internal operations;
- changed incidental formatting;
- replaced one correct algorithm with another; or
- refactored internal object structure without changing behavior.
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
## Expected effects of different changes
Use the following expectations when evaluating test failures and test maintenance:
| Change | Expected effect on tests |
|---|---|
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
## Separate mechanism from policy
Configurable thresholds and defaults must not be duplicated throughout the test suite.
For example, do not encode an internal concurrency limit indirectly:
```go
// Production policy:
const maxConcurrency = 4
// Brittle test:
err := startProcesses(5)
require.Error(t, err)
```
Instead, test the mechanism relationally:
```go
const limit = 2
runner := NewRunner(limit)
require.NoError(t, runner.Start(limit))
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
```
The test should prove:
- the configured limit is accepted; and
- one beyond the configured limit is rejected.
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
For concurrency limits, test both kinds of behavior when relevant:
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
## Avoid semantic duplication across layers
Each behavior should have a clear test owner.
- Parser tests own parsing cases.
- Validator tests own validation rules.
- Domain tests own transformations and invariants.
- Adapter tests own external integration behavior.
- Orchestrator tests own coordination and failure propagation.
- CLI tests own argument and configuration mapping.
- End-to-end tests prove that representative assembled workflows work.
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
## Use test doubles deliberately
Choose the least elaborate test double that provides the required control or observation.
As a default:
1. Prefer real collaborators when they are fast and deterministic.
2. Use small in-memory fakes when realistic stateful behavior is helpful.
3. Use stubs when a dependency only needs to provide controlled responses.
4. Use mocks when the interaction itself is contractual.
Mocks are appropriate when the contract includes facts such as:
- a notification is sent exactly once;
- a transaction is committed only after successful writes;
- cancellation reaches a subprocess;
- an expensive API is called no more than once; or
- a security audit event is emitted.
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
## Go-specific guidance
Use:
- table-driven tests for meaningful behavioral categories and boundaries;
- `t.TempDir()` for real filesystem behavior;
- `httptest.Server` for realistic HTTP interactions;
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
- golden files only when the complete output is intentionally stable;
- integration tests where correctness depends on component interaction; and
- a small number of representative end-to-end tests.
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
## Coverage
Coverage is a diagnostic, not a target.
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
## Regression tests
A bug fix should normally include a regression test that fails before the fix and passes afterward.
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
## Deleting or rewriting tests
Tests are maintained code, not permanent historical artifacts.
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
Strong candidates include tests that:
- require updates after harmless internal changes;
- directly assert private constants without protecting a real contract;
- duplicate the same policy across several layers;
- verify mock choreography rather than outcomes;
- snapshot large amounts of incidental output;
- test trivial private helpers already exercised through stable package behavior;
- protect risks already covered more effectively elsewhere;
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
- no longer correspond to a plausible failure mode.
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
## Reviewing a proposed test
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
1. What realistic defect would it catch?
2. How likely is that defect?
3. How consequential would it be?
4. Is the behavior already protected elsewhere?
5. At which layer should this behavior be owned?
6. Does the test assert a durable contract or an incidental implementation detail?
7. Could the implementation be refactored without changing the behavior and without editing this test?
8. What should cause this test to fail?
9. What legitimate changes should not cause this test to fail?
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
11. Is there a smaller or more direct test that protects the same risk?
Do not add the test when its expected lifetime cost exceeds its expected protective value.
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
## Definition of sufficient
A test suite is sufficient when:
- important contracts and invariants are protected;
- meaningful boundaries and failure modes are exercised;
- realistic and consequential regressions are credibly protected against silent recurrence;
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
- important external boundaries have realistic integration coverage;
- representative complete workflows are tested;
- failures provide useful signal rather than redundant noise;
- legitimate internal changes usually do not require test edits; and
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
The governing rule is:
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.

104
docs/release.md Normal file
View File

@@ -0,0 +1,104 @@
# Releasing Narratio
This document is the maintainer procedure for creating a Narratio source and
binary release. The synchronous release boundary is a successful push of one
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
change that result.
## Go prerequisite
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
asset construction; that asynchronous work does not change the tag-push
completion boundary.
## Choose a version and write its note
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
`vMAJOR.MINOR.PATCH`:
- increment `MINOR` for backward-compatible features;
- increment `PATCH` for backward-compatible fixes; and
- reserve a new `MAJOR` for an intentional breaking documented contract.
Before preparing the candidate, create
`docs/releases/vMAJOR.MINOR.PATCH.md` with this structure:
```markdown
# Narratio vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
The compatibility section identifies relevant CLI, configuration, artifact,
integration, or operating-contract changes. The upgrade section states the
required operator action, or explicitly says that no special action is
required. Release notes are immutable historical summaries; link to the
current canonical documentation for detailed behavior.
Commit the note and all candidate changes, then use the ordinary development
workflow to push that commit to `main`. Do not create a release tag before the
candidate is committed and `origin/main` contains the exact same commit.
## Validate the candidate
Run the shared checker from any directory:
```sh
scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
```
It validates the version and matching note, module hygiene, formatting,
whitespace, uncached tests, race tests, static checks, documentation, examples,
and three official cross-build assets. It uses `GOWORK=off`, does not contact
application services, CI, or Gitea, and does not create tags or modify tracked
source. Fix any failure on `main`, commit it, push it normally, and rerun the
checker.
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
`arm64`. The macOS cross-build proves compilation; it is not native macOS
runtime evidence.
## Publish the tag
From a clean checkout on `main` whose `HEAD` equals `origin/main`, run:
```sh
scripts/release.sh vMAJOR.MINOR.PATCH
```
The command fetches and checks `origin/main`, re-runs candidate validation,
then fetches and checks again before creating an explicitly unsigned lightweight
tag for the originally recorded commit. It refuses dirty, divergent, changed,
or already-tagged candidates. It pushes only:
```text
refs/tags/vMAJOR.MINOR.PATCH:refs/tags/vMAJOR.MINOR.PATCH
```
It never commits changes, pushes `main`, force-pushes, moves a tag, or pushes
all tags. A successful push of that exact ref completes the release command;
the command prints the tag and commit, then returns without waiting for CI,
querying Gitea, downloading assets, or checking checksums.
If a failure occurs before the tag is created, correct the candidate on `main`
and repeat validation. If the push fails after local tag creation, the local tag
is intentionally retained for inspection and the command must not be retried
blindly. Once the upstream tag has been pushed, it is immutable. Correct any
defect or failed asynchronous publication with a new patch version, a new
release note, and the complete procedure again.
## Optional asynchronous inspection
After a successful tag push, a human may later inspect the tag-triggered
Woodpecker run and the corresponding Gitea release for binaries and checksums.
This is optional follow-up only. Automated releasers must not wait for, poll,
or treat CI/Gitea completion as a condition of the successful tag push.

28
docs/releases/README.md Normal file
View File

@@ -0,0 +1,28 @@
# Release Notes
This directory contains immutable historical release notes for Narratio.
Future notes are created with the matching stable version and use this minimum
structure:
```markdown
# Narratio vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
See the [release procedure](../release.md) for creating a candidate and tag.
When asynchronous publication succeeds, the corresponding Gitea release is the
canonical source for downloadable binaries and checksums.
- [v2.0.0](v2.0.0.md)
- [v1.7.0](v1.7.0.md)
- [v1.6.0](v1.6.0.md)
- [v1.5.0](v1.5.0.md)

47
docs/releases/v1.5.0.md Normal file
View File

@@ -0,0 +1,47 @@
# Narratio v1.5.0
Narratio v1.5.0 makes repeated post-transcript artifact development faster and
more explicit while retaining the fixed, stage-driven pipeline model.
## Highlights
- The canonical pipeline now completes deterministic rendering before
extraction, cleanly separating transcript-generating stages from
artifact-generating stages.
- `narratio run` and `narratio session plan` accept inclusive `--from` and
`--through` bounds. Excluded transcript stages are not executed or
invalidated by a bounded artifact-regeneration run.
- `narratio regenerate-artifacts SESSION` is an exact convenience alias for a
forced run from `extract` through `analyze`, including focused
`--artifacts` selections.
- Configured Scriptorium artifacts now have independent,
manifest-authoritative freshness. Narratio reuses validated current work,
rebuilds stale prerequisites in dependency order, and persists successful,
failed, and newly stale artifact state when an analysis invocation only
partially succeeds.
- Publish consumes only configured artifacts backed by current manifest
evidence; incidental or tampered files are not promoted as current output.
## Reliability And Administration
- Bounded prerequisites are checked again under the session lock before any
run mutation, closing a concurrent-run race.
- Analysis fingerprints are stable across executable and configuration path
changes and continue to cover only Narratio-observable semantic inputs.
- Runner composition now carries one validated execution plan from command
parsing through prerequisite validation, adapter composition, manifest
recording, and stage execution.
- `narratio version` reports the exact tag embedded in official release
binaries; ordinary source builds report `dev`.
## Upgrade Notes
- Existing unbounded commands and direct `run-stage`, `analyze`, and `publish`
workflows retain their meanings.
- Manifests written before artifact-level analysis state remain readable.
Legacy aggregate analysis success is not sufficient freshness evidence, so
the first analysis evaluation after upgrading may regenerate configured
artifacts once.
- Narratio cannot observe executable contents or configuration, prompt,
profile, module, and other files loaded privately by Scriptorium. Explicitly
force affected artifacts after changing those private inputs.

74
docs/releases/v1.6.0.md Normal file
View File

@@ -0,0 +1,74 @@
# Narratio v1.6.0
Narratio v1.6.0 makes large pipeline configurations easier to organize,
inspect, and vary while adding character-oriented artifact generation and a
guarded, reproducible release procedure.
## Summary
Pipeline configuration can now be assembled from explicit additive imports and
a selected production or testing profile. Campaigns can own a canonical,
versioned party roster, and Scriptorium artifact families can expand one
definition into concrete per-character artifacts, dependencies, variables, and
publish rules.
New read-only configuration commands expose the fully resolved pipeline,
source provenance, semantic digest, and profile differences before a session is
run. Stage reuse now records configuration-sensitive semantic evidence so
profile or configuration changes cannot silently reuse incompatible work.
## Compatibility
This is a backward-compatible feature release. Existing monolithic pipeline
files, concrete Scriptorium artifacts, publish rules, and configurations
without profiles remain supported. When profiles are declared and no explicit
profile is selected, Narratio uses the configured production default.
An unversioned party file plus a separate players file remains available as an
isolated legacy compatibility path, but it cannot drive artifact families. New
campaigns and new party-oriented features should use the `narratio.party.v1`
schema. Existing manifests remain readable; missing legacy semantic evidence
is treated as stale rather than trusted.
Narratio continues to consume the documented Notarius D&D pipeline contract.
The release workflow still cross-compiles Linux, macOS, and Windows binaries
for `amd64` and `arm64`; cross-compilation is not native runtime evidence for
macOS or Windows.
## Upgrade
No special action is required for existing monolithic configurations that do
not adopt profiles or artifact families. On the first run after upgrading,
stages recorded by older manifests may regenerate once because those records do
not contain the new semantic configuration evidence.
To adopt the new configuration model, use the maintained
`examples/production-testing` bundle as a migration reference: split stable
settings into explicit imports, define a production default and optional
testing profile, convert campaign party data to `narratio.party.v1`, remove the
separate players file, and then introduce character artifact families. Review
the result with `narratio config validate`, `config show`, `config sources`, and
`config diff` before running a session.
Campaign, session, previous-session, and run identifiers must satisfy the
documented portable identity grammar. Existing manifests or remote state with
unsafe legacy identifiers must be migrated before use.
## Changes
- Added root-owned, non-recursive additive pipeline imports with strict,
source-aware conflict detection.
- Added named pipeline profiles with an explicit production default and
deliberate command-line selection.
- Added `config validate`, `config show`, `config sources`, and `config diff`
for read-only inspection of resolved configuration and provenance.
- Added the strict `narratio.party.v1` campaign roster, including stable
character IDs, player and character names, optional aliases, and classes.
- Added deterministic players derivation and canonical party delivery to
downstream integrations.
- Added character-oriented artifact families, corresponding member
dependencies, family selection, and generated publish policies.
- Added semantic configuration fingerprints and stage-specific resume checks
across transcript and artifact stages.
- Added shared release candidate, asset build, and guarded tag-publication
scripts, with tag-triggered publication remaining asynchronous.

100
docs/releases/v1.7.0.md Normal file
View File

@@ -0,0 +1,100 @@
# Narratio v1.7.0
Narratio v1.7.0 completes the maintainability refactor begun after v1.6.0. It
simplifies filesystem and process handling, makes resumable work more precise,
replaces legacy local and remote state with explicit authorities, and turns
cleanup into the final ordinary pipeline stage.
## Summary
The session manifest is now the sole durable local progress ledger. Analyze
checkpoints successful artifacts individually, allowing an interrupted or
partially failed analysis to resume at the first incomplete artifact without
rerunning successful siblings unless forced.
Remote publication now commits immutable run state before advancing one
verified current pointer. Publish locks are independent per-source objects;
locked outputs retain their previously committed declarations rather than
being overwritten or silently omitted. Local cleanup runs after notification
as a separately recorded, safely retryable stage.
The release also adopts Go 1.26, simplifies ordinary file operations around
Narratio's trusted group-owned workspace model, improves subprocess lifecycle
handling, decomposes several large orchestration and validation components,
and removes mutable global test seams.
## Compatibility
This is a minor release with deliberate removal of compatibility paths that
were deprecated and documented as subject to removal:
- Campaigns must use one versioned `narratio.party.v1` `party.yml`. Separate
players files, session-level party overrides, and unversioned party documents
are rejected.
- Session manifests must declare `schema_version:
narratio.session-manifest.v1` and satisfy the current strict schema. Older
unversioned state and manifests containing the retired
`post_publish_cleanup` obligation are rejected rather than migrated.
- Remote current state must be selected by
`current/commit-pointer.json` and its referenced immutable run commit. Legacy
current-state layouts are no longer inferred.
- The retired aggregate `locks.yml` object is ignored. Publish locks now live
as independent strict objects under the session lock prefix.
- Run manifests are no longer produced or consumed. The session manifest owns
durable stage progress; invocation summaries and run workspaces remain
operational data rather than a second progress authority.
The CLI and current pipeline/configuration formats otherwise remain compatible.
Building and maintaining Narratio now requires Go 1.26. Cross-built Linux,
macOS, and Windows assets remain available for `amd64` and `arm64`.
## Upgrade
Before upgrading a campaign that still uses compatibility data:
1. Replace legacy party/player inputs with the documented
[`narratio.party.v1`](../integrations/party.md) campaign roster.
2. If a local manifest contains an outstanding `post_publish_cleanup`
obligation, complete it with the pre-upgrade binary. Otherwise verify the
remote commit, move aside or regenerate the incompatible manifest, or
restore a verified committed snapshot. Do not migrate state by manually
editing JSON.
3. Republish sessions that do not yet have a valid immutable run commit and
`current/commit-pointer.json`; legacy remote layouts are not current state in
v1.7.0.
4. Recreate any required publication locks with `narratio session locks add`.
After verifying the new locks and publication behavior, the ignored legacy
`locks.yml` object may be removed manually.
Existing compatible session manifests and already committed remote snapshots
need no special action. Consult [Operations](../operations.md) and
[Troubleshooting](../troubleshooting.md) for recovery procedures.
## Changes
- Upgraded the maintained development and release baseline to Go 1.26.
- Reworked subprocess execution and Windows process-tree supervision while
preserving contextual wait and cleanup failures.
- Replaced custom confined ordinary I/O and directory-promotion machinery with
simpler standard-library operations while retaining bounded reads, atomic
single-file visibility, workspace modes, validated downloads, and
root-confined destructive operations.
- Removed legacy party, manifest, artifact-discovery, remote-current, and lock
compatibility paths.
- Consolidated durable progress in the session manifest and removed the run
manifest domain model.
- Unified runner planning and execution decisions, separated invocation
lifecycle responsibilities, and avoided creating empty run workspaces for
fully reused or cleanup-only invocations.
- Added per-artifact analyze execution, checkpointing, reconciliation, and
recovery.
- Decomposed configuration validation by domain and made validation diagnostics
deterministic without treating incidental ordering or wording as a contract.
- Introduced immutable remote publish commits, one final current pointer,
explicit output planning, safe cross-run carried outputs, and independent
per-artifact remote locks.
- Added `cleanup` as the final canonical stage after `notify`, with validated
publish evidence, safe retry, explicit selection, and no separate cleanup
obligation.
- Replaced mutable command test hooks with invocation-scoped dependencies and
improved structural test diagnostics.

94
docs/releases/v2.0.0.md Normal file
View File

@@ -0,0 +1,94 @@
# Narratio v2.0.0
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
simplifies the supported release platforms, and makes artifact definitions and
their effective dependency graph directly inspectable.
## Summary
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
imports, fixed-path named profiles, provenance, defaults, and typed credential
references. The recommended production layout keeps production values in the
base, defines each ordinary artifact and artifact family in its own imported
file, and keeps testing changes in `profiles/testing.yml`.
Configured-artifact inputs now create their dependency edges automatically.
`narratio config artifacts` reports the finalized concrete catalog, including
generated party-family members, without loading a session or initializing
runtime adapters.
## Compatibility
This is a breaking configuration and platform release. There is no runtime
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
presence/default behavior. Defaults apply only to absent values; explicit empty,
zero, false, and null values retain their typed meaning and may fail Narratio
domain validation.
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
and macOS amd64 builds are no longer produced or supported. Existing v1 release
notes remain historical descriptions of their releases.
Inspection commands still avoid session, manifest, workspace, adapter, and
remote-service activity, but they are not credential-free loaders: every
explicit typed credential reference in the selected effective configuration must
resolve; references replaced or removed by that profile need not resolve.
Pipelines containing artifact families also require a campaign so the canonical
party can be expanded.
`config show` is a secret-safe review projection, not a promised reloadable
authoring file.
## Upgrade
Migrate each operational bundle before running v2:
1. Replace old composition syntax with root-owned
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
additive base sources; they do not override one another.
2. Fold the old production overlay into the base pipeline. Move testing changes
to `profiles/testing.yml`; compare the two independent resolutions with
`narratio config diff default testing`.
3. Follow the maintained
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
stable artifact identity.
4. Move credential files to an untracked `secrets/` directory beside
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
5. For static S3 credentials, configure both S3 references (commonly
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
To use instance roles, workload identity, shared SDK files, or another AWS SDK
discovery provider, omit both references. A partial pair is invalid.
6. Remove a `depends_on` entry when it only duplicates a
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
without supplying the upstream output as an input.
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
that previously acquired defaults after being explicitly empty. Do not use
`config show` output as a source migration mechanism.
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
with the same profile, campaign, and secrets selection intended for runtime.
Resolve the campaign-owned party whenever families are present.
9. Update deployment and download automation to one of the three maintained
platform assets before removing the v1 binary.
The [configuration reference](../config.md), [CLI reference](../cli.md), and
[maintained examples](../../examples/README.md) are the canonical detailed
guides.
## Changes
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
while keeping Narratio domain validation and post-campaign expansion explicit.
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
unified safe effective show/diff/digest projection, and source provenance.
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
Audita credential references without exposing resolved values.
- Inferred, normalized, and validated configured-artifact input dependencies as
part of the one concrete graph consumed by analyze planning and resume logic.
- Added `config artifacts` with stable concrete identities, definition sources,
effective dependencies, and generated family/party provenance.
- Added the maintained production-base/testing-overlay artifact authoring layout.
- Reduced release assets and active platform support to Linux amd64/arm64 and
macOS arm64.

70
docs/releases/v2.1.0.md Normal file
View File

@@ -0,0 +1,70 @@
# Narratio v2.1.0
This release adds explicit Notarius credential forwarding and cache reuse,
with predictable forced regeneration and clearer artifact-development guidance.
## Summary
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
its accepted execution checkpoints. Stable, verified reference snapshots allow
unchanged references to retain their cache identity across Narratio invocations.
Configuration loading now uses ConfigKit v0.3.0.
## Compatibility
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
fields are optional. Existing v2 configuration needs no schema migration.
Configured credential references must resolve during configuration loading,
even when extraction will not execute; omitted references add no requirement.
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
refresh. This includes `regenerate-artifacts`, which remains an alias for a
forced run from extract through analyze. `analyze` continues to force analysis
only, preserving extraction results and transcripts.
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
minimum-version gate. The executable must support the flags selected for the
invocation, including `--chunk_cache refresh` for forced extraction. Reference
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
schemas are unchanged; reused producers may report zero production attempts.
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
arm64. macOS validation is cross-compilation, not native runtime testing.
## Upgrade
No special migration is required for existing v2 configuration. To use the new
features:
- Declare `notarius.llm_api_key` with the environment-variable name expected by
the deployed Notarius profile, supplying its value through the existing
environment or secrets-directory mechanism.
- Enable checkpoint recording in Notarius configuration before setting
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
roots for service deployments. An optional `notarius.chunk_cache` overrides
its ordinary chunk-plan policy.
- Expect an initial checkpoint miss when moving from old run-local reference
paths to stable session snapshots. No migration or deletion of old caches is
required. Force refreshes the current input's plan, not the whole cache.
- Use `narratio analyze SESSION` for analysis-only iteration, and
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
See [Configuration](../config.md), the
[Notarius integration contract](../integrations/notarius.md), and
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
for current details.
## Changes
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
from machine configuration.
- Added optional typed Notarius credentials, exact-name environment delivery,
safe configuration projections, and captured-stream redaction for custom names.
- Added Notarius resume and chunk-cache controls with force taking precedence.
- Added verified, content-addressed session reference snapshots that survive
ordinary run cleanup without becoming published artifacts.
- Preserved relative-workspace support for snapshot delivery and confinement.
- Kept credentials and cache-policy controls outside extract's semantic resume
identity, so operational changes alone do not invalidate completed results.
- Added an opt-in real Notarius integration test using a local provider to
verify reuse, partial-failure recovery, and forced refresh without paid calls.

33
docs/releases/v2.1.1.md Normal file
View File

@@ -0,0 +1,33 @@
# Narratio v2.1.1
This release fixes configuration loading for publish rules that explicitly set
requiredness while relying on a derived destination.
## Summary
Publish rules with a derivable source and omitted `dest` now load correctly
whether `required` is omitted, explicitly true, or explicitly false. Previously,
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
diagnostic instead of accepting valid configuration.
## Compatibility
This is a backward-compatible bookkeeping correction. Publish schemas,
destination derivation, requiredness defaults, and YAML syntax support are
unchanged. Explicit destinations and explicit false values remain intact.
No ConfigKit upgrade or publication/storage behavior change is involved.
## Upgrade
No special upgrade action or configuration migration is required. Operators
need not add a derivable `dest`, remove explicit `required` values, or change
flow-style YAML to block-style YAML. See the
[configuration reference](../config.md#publish-configuration-summary).
## Changes
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
while retaining field-level resolution records.
- Added real load/validation and safe effective-configuration regression coverage
for built-in and configured-artifact sources, explicit requiredness, and an
explicit destination that must remain unchanged.

26
docs/releases/v2.1.2.md Normal file
View File

@@ -0,0 +1,26 @@
# Narratio v2.1.2
This release allows secrets directories to contain hidden metadata without
preventing credential loading.
## Summary
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
valid credentials from loading and are omitted from bootstrap reports.
## Compatibility
This is a backward-compatible dependency bug fix. Configuration schemas,
existing-environment precedence, and validation of non-hidden entries are
unchanged. Hidden entries still count toward the directory entry limit.
## Upgrade
No special upgrade action or configuration migration is required. See the
[credential configuration reference](../config.md) for secrets-directory usage.
## Changes
- Updated ConfigKit from v0.3.0 to v0.3.1.
- Clarified hidden-entry handling in the secrets-directory documentation.

53
docs/releases/v2.2.0.md Normal file
View File

@@ -0,0 +1,53 @@
# Narratio v2.2.0
This release adds resumable per-speaker transcription, replayable WhisperX
uploads, and explicit Notarius configuration profile selection.
## Summary
Successfully checkpointed speaker transcripts survive later transcription
failures. Ordinary retry reuses valid completed speakers and processes only the
remaining work. WhisperX uploads can now be replayed by the HTTP transport
without buffering whole audio files.
## Compatibility
Existing CLI commands, speaker identities, transcript formats, and pipeline
ordering are unchanged. Session manifests gain optional per-speaker completion
evidence. Older manifests remain readable, but older Narratio binaries may
reject manifests containing the new fields; downgrading after transcription
is not transparent.
The optional `pipeline.notarius.config_profile` field selects a downstream
Notarius configuration profile independently of Narratio's own profile. Using
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
supporting version. Omitting the field retains downstream default selection.
## Upgrade
No configuration migration is required. When transcribe is next selected for
an older aggregate-only manifest, all current speakers are transcribed once
to establish reusable evidence. Previously unrecorded results from failed runs
are not automatically imported.
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
reuse of prior results; an ordinary retry after a failed forced invocation can
retain replacements already checkpointed by that attempt. See the
[operations guide](../operations.md) and
[Notarius profile configuration](../config.md#notarius-configuration-profile).
## Changes
- Persist validated speaker completions independently of aggregate stage status,
with serialized saves for concurrent work and input/output integrity checks.
- Rebuild aggregate transcript references when speakers are removed or those
references need repair, without retranscribing otherwise-current speakers.
- Keep removed and incomplete speaker results out of versioned merge inputs.
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
service rejection still fails normally.
- Provide fresh streaming multipart bodies for HTTP transport replay while
preserving application retries, cancellation, and resource cleanup. This
addresses the missing-body-replay limitation, not persistent upstream
protocol failures or exactly-once service execution.
- Pass explicit Notarius configuration profiles through the CLI and include
their selection in extraction reuse decisions.

View File

@@ -0,0 +1,102 @@
---
title: Select a Notarius configuration profile through Narratio
status: proposed
consumer: rakestraw.dnd Ansible collection and its pipeline role
use_case: >-
The Ansible pipeline role deploys Narratio and Notarius as independently
configured applications. Operators select Narratio's testing profile to
develop prompts and other pipeline assets with inexpensive models while
retaining production settings in the base configuration. That selection
currently cannot select a Notarius configuration profile, leaving extraction
on its production settings.
desired_outcome: >-
Narratio's testing overlay can explicitly select Notarius's testing overlay.
Notarius owns the resulting model choices and any other testing-specific
settings. The operator selects testing once when invoking Narratio, without
editing production files or separately invoking Notarius.
rationale: >-
Both applications already use ConfigKit configuration overlays. Selecting
the downstream overlay preserves each application's configuration ownership
and supports future Notarius testing adjustments without adding a Narratio
configuration field for every downstream setting. This is explicit profile
composition, not a requirement for arbitrary recursive orchestration.
requested_interface:
field: pipeline.notarius.config_profile
value: Optional Notarius ConfigKit configuration profile name
supplied: >-
Invoke Notarius with --config-profile and the selected name when extraction
executes. The setting is available in Narratio's base configuration and
its ordinary configuration overlays.
omitted_or_null: >-
Do not supply a profile-selection flag. Preserve existing Notarius
configuration selection behavior.
downstream_semantics: >-
Retain Notarius's existing selector semantics, including the default
selector for its base configuration. Notarius resolves profiles relative
to its own configuration root, not Narratio's root.
explicit_selection: >-
Do not automatically forward Narratio's profile name. The selected
Notarius profile may have the same name or a different name.
distinction: >-
This selects a ConfigKit configuration overlay through --config-profile,
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
can itself select the desired LLM profiles using native Notarius settings.
illustrative_consumer_configuration:
note: >-
These are separate files managed by the Ansible suite, not a proposed
combined configuration document. The Notarius pipeline and LLM profile
names are illustrative; the referenced LLM profile must be available to
Notarius.
narratio_profiles_testing_yml: |
notarius:
config_profile: testing
notarius_profiles_testing_yml: |
pipelines:
dnd-session:
llm_profile: gemini-flash-latest
operator_workflow: >-
Select --profile testing on the ordinary Narratio invocation. Narratio
selects Notarius's testing configuration overlay when it runs extraction;
other Narratio testing overrides continue to apply normally.
observable_behavior:
- >-
Notarius owns overlay discovery, composition, validation, and the effective
downstream settings. A missing or invalid selected overlay must not silently
fall back to production or be reported as successful extraction.
- >-
Changing the selected Notarius configuration profile must not silently
reuse Narratio's completed extraction from the previous selection.
Existing downstream invalidation behavior applies when extraction changes.
- >-
Editing Notarius overlay contents behind an unchanged selector retains
the existing operator responsibility to force extraction when needed.
This request does not require inspecting or tracking private Notarius files.
- >-
Existing credential forwarding remains independently configurable through
notarius.llm_api_key, including in the Narratio testing overlay when the
selected Notarius settings require a different credential reference.
consumer_follow_up: >-
After Narratio releases this interface, the standalone Narratio Ansible role
can expose an optional variable and omit the field when unset. The suite role
can deploy Notarius's testing overlay and select it from Narratio's testing
overlay, then distribute the changes through the collection. These consumer
updates are separate from this upstream feature request.
scope_boundaries:
- No Notarius change is currently required; v0.7.0 supports --config-profile.
- No separate Narratio-side Notarius LLM-profile selector is requested.
- No automatic profile-name propagation or downstream configuration rewriting.
- No new credential discovery, copying, or provider-name remapping.
- This request specifies consumer behavior, not internal Narratio code changes.
references:
narratio_configuration: ../config.md
narratio_notarius_integration: ../integrations/notarius.md
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md

View File

@@ -0,0 +1,171 @@
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
Status: Implemented; release target v2.1.1
## Summary
Narratio should successfully derive a publish destination when a valid
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
This is a configuration-resolution bookkeeping defect, not a YAML formatting
restriction. The fix should preserve the existing public configuration
contract: operators may omit a derivable destination and may independently set
`required` to either `true` or `false`.
## Observed Behavior
A pipeline containing a rule equivalent to the following fails during native
configuration loading:
```yaml
publish:
outputs: [{"source": "narratio.transcript.final", "required": true}]
```
Narratio reports:
```text
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
```
The same failure is possible with ordinary block-style YAML when `required` is
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
cause of the defect.
The existing destination-derivation tests do not expose this case because
their rules also omit `required`. Narratio then supplies the default value for
`required` and records a change covering the complete `publish.outputs`
sequence. That broader record happens to cover the destination mutation as
well, masking the missing resolution record that occurs when `required` is
already explicit.
## Expected Contract
The current configuration contract remains unchanged:
- `publish.outputs[].source` is required;
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
selected built-in or configured artifact source;
- `publish.outputs[].required` defaults to `true`, but an operator may set it
explicitly to either `true` or `false`; and
- equivalent valid YAML representations have equivalent configuration
semantics.
For example, both rules below should load and validate, retain their explicit
requiredness, and receive the canonical derived destination:
```yaml
publish:
outputs:
- source: narratio.transcript.final
required: true
- source: narratio.transcript.final_markdown
required: false
```
## Root Cause
ConfigKit intentionally treats sequences as atomic when comparing values
before and after application resolution. Changing a field within one
`publish.outputs` entry is therefore observed as a change to the
`/publish/outputs` sequence.
Narratio currently derives each missing destination by recording a
normalization at the individual `publish.outputs[index].dest` path. That child
record does not cover the atomic parent-sequence change. A separate
container-level record is currently created only when Narratio also defaults
an omitted `required` value, so configurations with explicit `required` values
lose that incidental coverage.
## Recommended Fix
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
atomic-sequence contract. Whenever resolution derives one or more destinations
inside `publish.outputs`, the recorded resolution changes must cover the
`/publish/outputs` container. Destination derivation must not depend on an
unrelated `required` default being applied in the same list.
The implementation may retain more specific provenance where useful, but it
must provide valid container-level coverage for the sequence mutation. Do not
change ConfigKit's sequence comparison semantics merely to accommodate this
call site.
## Scope
- Correct destination derivation for publish rules that explicitly set
`required` and omit a derivable `dest`.
- Preserve explicit `required: true` and `required: false` values.
- Preserve canonical destination derivation for built-in transcript sources
and configured artifact sources.
- Add focused regression coverage through Narratio's real configuration load
and validation boundary.
- Prepare a backward-compatible patch release after the fix passes the normal
repository validation.
## Non-goals
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
destination derivation rules.
- Requiring operators or configuration generators to supply `dest` when it is
derivable.
- Requiring operators to omit explicit `required: true` values.
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
- Changing ConfigKit's documented atomic treatment of sequences.
- Adding storage, publication, or network integration behavior.
## Regression Coverage
Add one focused package-level behavioral test that loads and validates a
pipeline with omitted destinations and explicit requiredness. A representative
list should include both `required: true` and `required: false`, then assert
that:
- loading and validation succeed;
- each destination is derived from its source; and
- each explicit `required` value is preserved.
Using the compact flow-style representation that exposed the integration
failure is reasonable, while the existing block-style cases continue to cover
omitted `required`. The test should exercise the public configuration behavior,
not ConfigKit call choreography or Narratio's private recording mechanism.
No network service, object store, credential, or test double is required.
## Target End State
All valid publish rules with derivable sources behave consistently across
these cases:
| `dest` | `required` | Result |
| --- | --- | --- |
| omitted | omitted | destination derived; required defaults to `true` |
| omitted | `true` | destination derived; explicit `true` retained |
| omitted | `false` | destination derived; explicit `false` retained |
| explicit | either value | explicit destination and requiredness retained |
`narratio config validate`, `narratio config show`, and ordinary pipeline
loading accept the corrected cases without an unrecorded-change diagnostic.
No operator configuration migration or workaround is required.
## Release
This is a backward-compatible correction to documented configuration behavior
and should be released as Narratio `v2.1.1`. The release note should identify
the corrected explicit-required/derived-destination interaction and state that
no special upgrade action is required.
## Acceptance Criteria
- The focused regression test fails against v2.1.0 behavior and passes after
the fix.
- Publish rules with explicit `required: true` or `required: false` and omitted
derivable destinations load and validate successfully.
- Derived destinations and explicit requiredness appear correctly in the safe
effective configuration.
- Existing destination derivation, publish validation, imports, profiles, and
configuration inspection tests remain green.
- The complete repository validation and release-candidate checks pass.
- The correction is published as the immutable `v2.1.1` tag with the required
release note.

View File

@@ -0,0 +1,431 @@
# Implementation Plan: Speaker Checkpoints and Replayable Uploads
Status: Implementation complete; Stages 1-7 complete.
## Scope and Instructions
Implement [the feature roadmap](transcribe-checkpoints-and-replay.md). This plan
replaces the completed Notarius-profile plan; that work remains in Git history.
Complete the numbered stages in order, one bounded implementation prompt per
stage for gpt-5.6-terra. Record completion and actual verification results here.
Intermediate stages are not a finished feature and must not be released alone.
Before working, read `AGENTS.md`, `docs/development.md`, all three policies in
`docs/policy/`, the roadmap, and the stage-specific preparation below. Use the
maintained Go 1.26 toolchain. Preserve operator choices, speaker filenames,
bounded concurrency, streaming, and existing credentials/settings. Do not add
new commands, dependencies, service requirements, arbitrary speaker limits,
strict transcript schemas, or a generic checkpoint framework. No release or
upstream service work is part of this plan.
## Settled Design Decisions
### Progress Representation and Recovery
Add optional `transcribe_state_version` (version 1) and `transcribe_speakers`
to `manifest.StageRecord`. The latter is a map keyed by the existing derived
speaker ID, containing completed results only. Absence means that work must be
done; a separate pending/running/failed state machine is unnecessary. Aggregate
stage status/error remains authoritative for invocation failure. A versioned
empty collection is meaningful: no earlier completions are reusable.
Use a typed `TranscribeSpeakerRecord` with fingerprint version/digest,
session-relative audio path, audio SHA-256 and size, one `ArtifactRecord` output
of kind `transcript_raw`, output size, and producer run ID. The output carries
its session-relative canonical path and SHA-256. No new transcript source IDs,
synthetic schema contracts, timestamps, or historical attempt collections are
needed. Existing aggregate diagnostics can retain attempt/status/duration data
for work performed in the current invocation; do not fabricate that information
for reused outputs.
Keep the session-manifest schema identifier unchanged for this optional additive
state; validate supported per-stage versions and retain strict decoding. New
Narratio reads old manifests. Old binaries may reject the added fields under
their existing strict decoding; document this downgrade limitation.
For pre-feature manifests with no speaker evidence, regenerate transcription
once when transcribe is next selected, even if the aggregate status succeeded.
Old records do not bind individual transcript results to the audio bytes they
used. Do not guess that association from current files, old `per_file` metadata,
or abandoned run directories. This is the conservative upgrade path, not a new
general migration system. No other stage or unselected range is forcibly run.
### Reuse and Persistence
Compute a versioned per-speaker fingerprint from speaker ID, session-relative
audio identity, actual audio SHA-256/size, and the existing transcribe semantic
configuration fingerprint (URL and language). Do not include invocation ID,
absolute workspace root, concurrency, retry settings, timeout, or credentials.
Stream input hashing with cancellation; do not trust stored input checksums
alone. Recheck input bytes before accepting a new result to catch changes
observed during transcription; fail that job without checkpointing mismatched
evidence. This is not a hostile-filesystem snapshot or concurrent-editor
transaction guarantee and requires no extra audio copies or file locks.
Validate reusable output identity, canonical location, JSON, checksum, and size.
Use existing confinement/file primitives. Missing or corrupt regular outputs
become work; unsafe paths/non-regular conflicts follow existing safety errors,
not recursive deletion. Missing required audio is an input error, not permission
to reuse an unverifiable transcript. Removed inputs are removed from the current
collection and aggregate outputs without deleting arbitrary files.
One stage-owned mutex serializes candidate collection construction, manifest
save, and installation of committed state. Save a detached candidate and only
then update live authority, following analyze's pattern. Workers must not mutate
the live manifest outside that serialized section. Keep expensive HTTP work
outside the mutex. Stop dispatch/acceptance on checkpoint failure, join workers
before return, and preserve the last successfully committed collection.
Before launching requests, durably checkpoint the reconciled collection. For
force, that collection is empty: this prevents later ordinary retries from
resurrecting pre-force outputs. If this initial save fails, do not start requests
or overwrite outputs. Force again clears the current collection again; ordinary
retry retains replacements already checkpointed after the previous force.
Save each validated result at its canonical raw-transcript location before
checkpointing it. Run-local files remain staging/diagnostics, not reuse
dependencies. Bytes left by an unsuccessful checkpoint have no completion
authority. A worker may checkpoint a completed result using the parent stage
context even if a sibling canceled the work context; parent cancellation or a
checkpoint failure may prevent that save. Guarantee retention of committed
work, not all responses that happened to finish during cancellation.
### Multipart Replay Mechanism
Replace the pipe/producer upload mechanism with a small pull-based multipart
body backed by the audio reader. Use `mime/multipart.Writer` to encode a bounded
metadata prefix and suffix around the file payload: capture the prefix after
`CreateFormFile`, then encode the language field and closing boundary as the
suffix. Generate this framing once per application attempt. Each body combines
fresh readers for prefix, freshly opened audio, and suffix using `io.MultiReader`.
No full-audio buffer or temporary multipart spool is needed.
The initial body and `Request.GetBody` use the same factory/framing. Each owns
an independent audio handle. Give bodies idempotent Close and context-triggered
closure (for example `context.AfterFunc`), without holding a mutex across a
blocking Read. Read/Close coordination must release a blocked source and retain
genuine source-read/close errors while not treating deliberate early closure as
source failure. The attempt owns cleanup of all created bodies, including
replays; reject new body creation after its context expires.
Remove the old producer `Wait` dependency. Track body-local source errors with
synchronization so a swallowed upload read error cannot become a successful
transcript, and a deliberately closed original body cannot poison a successful
replay. Keep the outer retry policy and per-attempt context. Do not add replay
counters to the user retry budget or depend on Go's exact internal retry count.
Preserve ordinary HTTP redirect policy; body replay can also enable Go's
body-preserving redirects. Do not introduce a new redirect restriction.
## Stage 1: Add Durable Speaker Completion State
Status: Complete.
Completion marker (2026-09-21): Added optional versioned transcribe-owned
per-speaker completion records, strict load/save validation, isolated collection
cloning, legacy aggregate-only compatibility, and lifecycle-preservation
coverage. Verified with `go test ./internal/manifest` and `git diff --check`.
Preparation: `internal/manifest/manifest.go`, `store.go`, `analyze_state.go`,
their tests, and `docs/internal/manifest.md`.
1. Implement the additive fields, record type, version constants, validation,
and deep-copy helper described above in a focused transcribe state file.
2. Validate ownership by the transcribe stage, supported versions, required
completed-record evidence, output kind, safe relative paths, SHA-256 values,
sizes, and producer identity consistency. Use existing speaker IDs as-is;
do not apply analysis-artifact key regexes or copy unrelated analysis limits.
3. Integrate validation with both load and save. Preserve speaker state through
running/failed/stale transitions and aggregate result clearing. Generic stage
invalidation does not discard checkpoints; reconciliation checks their data.
4. Test round-trip populated and versioned-empty collections, old manifests
without fields, malformed evidence, deep-copy isolation, and lifecycle
preservation. Avoid imposing JSON formatting or map order as contracts.
Verify: `go test ./internal/manifest`.
Done when durable storage can represent partial completion independently of
aggregate status, without activating new stage behavior yet.
## Stage 2: Implement Read-Only Speaker Reconciliation
Status: Complete.
Completion marker (2026-09-21): Extracted shared deterministic transcribe job
construction and added read-only, cancellation-aware per-speaker input and
output reconciliation. Covered unchanged reuse, semantic and operational
settings, changed/added/removed inputs, legacy and forced work, missing/corrupt
outputs, unsafe evidence, and missing audio without connecting runtime skip or
checkpoint behavior. Verified with
`go test ./internal/stage ./internal/manifest` and `git diff --check`.
Preparation: `internal/stage/transcribe.go`, `semantic_contracts_initial.go`,
`analyze_reconciliation.go`, `extract_resume.go`, their tests, and the artifact
path/integrity helpers they use.
1. Extract the existing audio discovery/job construction into a shared,
transcribe-local reconciliation path. Preserve current input selection and
speaker identity behavior. Build sorted jobs and classify completed results
against the current input set and evidence defined above.
2. Add pure fingerprint construction and streamed input/output inspection.
Resolve manifest input paths with existing session-relative/absolute reading
helpers; do not change input precedence or assume the current working directory
is the session root. Newly written checkpoint paths are always relative.
3. Return current records and work-to-do without writing files or manifest state.
Force selects all work and an empty starting collection; old unversioned
transcribe state supplies no reusable records. Do not connect aggregate skip
validation until execution can produce the new evidence in Stage 3.
4. Test unchanged reuse; one changed audio among unchanged speakers; URL/language
changes; operational-setting exclusions; added/removed speakers; missing and
corrupt regular outputs; unsafe record paths; missing audio; and force.
Prefer compact tables and real temporary files over mocked hashing.
Verify: `go test ./internal/stage ./internal/manifest`.
Done when one read-only decision path describes exactly which speakers need
work, without altering existing runtime behavior prematurely.
## Stage 3: Execute and Checkpoint Speakers Incrementally
Status: Complete.
Completion marker (2026-09-21): Connected reconciliation to bounded execution,
durably checkpointed the starting and per-speaker collections through detached
manifest candidates, preserved committed work across failures and cancellation,
rebuilt deterministic aggregate outputs from reusable evidence, and added
speaker-aware resume validation. Updated production-style fixtures and covered
serialized concurrent saves, force/retry replacement, input rechecks, save
failures, partial recovery, and aggregate reconstruction. Verified with
`go test ./internal/stage ./internal/app`, `go test -race ./internal/stage`, and
`git diff --check`.
Preparation: `internal/stage/transcribe.go`, `analyze_execution_state.go`,
`run_local.go`,
`internal/app/runner.go`, and transcribe/checkpoint test fixtures.
1. Wire reconciliation into Run, require the existing ManifestStore collaborator,
and checkpoint the starting collection before launching HTTP work. Implement
detached candidate saves and serialized live-state updates. Audit Store.Save's
normalization side effects so a failed candidate save cannot mutate shared
authority; use detached records/collections wherever it may write.
2. Retain bounded errgroup execution and fail-fast cancellation. Check the work
context before beginning each queued request. After success, validate adapter
output path/JSON, recheck audio evidence, materialize canonical bytes, compute
output evidence, and checkpoint the new record. Check a checkpoint-failure
latch under the serialization lock before accepting further completions.
3. Preserve committed successes on all error/cancellation exits, including
starting-save, canonical-write, per-speaker-save, and aggregate-save failures.
Return meaningful speaker/error context and let the runner retain its existing
terminal failure behavior. Do not report partial aggregate outputs as success.
4. Assemble final outputs in deterministic speaker order from the reconciled
completed records, including reused speakers, with original producer IDs and
checksums. If all records are current but aggregate status is failed/running,
finish without WhisperX calls and rebuild the aggregate success.
5. Implement `transcribeStage.ValidateResume` using the same read-only
reconciliation, so a succeeded stage cannot skip changed audio or invalid
transcripts. Retain the existing URL/language semantic check and runner
invalidation behavior; no whole-config digest invalidation.
6. Adapt existing tests that require no canonical partial files: the new contract
permits checkpointed files but forbids aggregate success. Update fixtures to
provide real manifest stores and records rather than a production fallback
that silently disables checkpointing.
7. Test A/B completion followed by C failure and D unattempted (concurrency 1),
reload and retry only C/D; concurrently finishing speakers without lost saves;
failure and cancellation after a checkpoint; save failure retaining earlier
authority; and force failure followed by ordinary retry without old-result reuse.
Verify: `go test ./internal/stage ./internal/app` and
`go test -race ./internal/stage`.
Done when incremental progress and aggregate success work together safely under
ordinary, concurrent, canceled, and forced execution.
## Stage 4: Verify Runner, Merge, and Durable Lifecycle Integration
Status: Complete.
Completion marker (2026-09-22): Added real-persistence application coverage for
cross-invocation retry, interrupted running state, failed aggregate saves,
force/reset persistence, downstream invalidation, and one-time legacy upgrade.
Versioned merge input discovery now requires aggregate success and an exact
match between current speaker evidence and declared aggregate outputs, while
legacy/manual directory fallback remains available. Restore coverage preserves
session-relative speaker paths, leaves unpublished raw transcripts absent, and
canonical transcripts remain reusable after run-directory cleanup. Verified
with `go test ./internal/app ./internal/stage ./internal/manifest`,
`go test -race ./internal/app`, and `git diff --check`.
Preparation: `internal/app/runner.go`, `runner_runtime.go`,
`analyze_checkpoint_recovery_test.go`, `runner_terminal_failure_test.go`,
`restore_manifest_paths.go`, `internal/stage/merge.go`, and existing publish,
restore, and cleanup tests routed from `docs/internal/overview.md`.
1. Add application-level tests with real manifest persistence and fake WhisperX:
failed stage across separate invocations; all speakers committed before a
failed aggregate save; interrupted running state; force/reset persistence;
and unchanged upstream/succeeded downstream invalidation behavior. Inject
errors at durable boundaries, not every private method call.
2. Close `discoverRawTranscripts`' directory-scan fallback for versioned
transcribe state: incomplete aggregate state is not usable, and completed
state consumes only declared current aggregate outputs. Keep legacy/manual
fallback behavior for manifests without the new state rather than banning
previously allowed standalone workflows. Test that partial files and removed
speaker files cannot leak into a versioned merge, including direct stage calls.
3. Audit manifest copies/projections, remote manifest publication, restored path
handling, and cleanup. Preserve relative speaker paths on restore and verify
they resolve under the destination session, not the producer machine. Reuse
existing confinement validation, not a second path-policy layer.
4. Add focused round-trip coverage through existing publish/restore fixtures for
speaker records. Restored outputs absent from the published selection are
non-reusable locally; do not silently expand remote publication to every raw
speaker or upload audio as a new default. Removing run directories must not
remove canonical checkpointed transcripts or make reuse require run files.
5. Verify old succeeded manifests are readable and rerun once when transcribe is
selected, then reuse normally. Keep old manifests usable for other commands
and bounded ranges. Do not automatically import old unrecorded JSON files.
Verify: `go test ./internal/app ./internal/stage ./internal/manifest` and
`go test -race ./internal/app`.
Done when the new authority survives runner and storage lifecycles and cannot
expose incomplete transcripts downstream.
## Stage 5: Implement Replayable Streaming Multipart Bodies
Status: Complete.
Completion marker (2026-09-22): Replaced the producer-goroutine/pipe upload
with bounded standard-library multipart framing and independently opened,
pull-based streaming bodies shared by the initial request and `GetBody`.
Bodies are tracked, idempotently closed, canceled with their attempt context,
and retain genuine read/close failures without treating an intentionally
abandoned original as a replay failure. Added focused coverage for identical
framing, partial-read restart, simultaneous handles, source failures, and
blocked-reader release. Verified with `go test ./internal/adapters/whisperx`,
`go test -race ./internal/adapters/whisperx` (including ten repeated runs), and
`git diff --check`.
Preparation: `internal/adapters/whisperx/http.go`, `http_test.go`,
`docs/integrations/whisperx.md`, and the installed Go net/http Request/GetBody
contract. Keep this change inside the WhisperX adapter.
1. Implement the pull-based framing/body factory described above, using standard
multipart encoding rather than hand-escaped MIME headers. Add no new payload
cap or upload-size setting. Preserve file-before-language wire semantics and
the existing form names.
2. Install the initial body and GetBody with the same per-attempt boundary.
Track and close all opened bodies even if a custom transport returns early;
make cleanup/context callbacks safe against overlapping reads and closes.
A GetBody failure must be returned, not converted into an empty upload.
3. Replace pipe-specific Wait/error paths in doTranscribeAttempt with source-error
handling appropriate to pull bodies. Genuine source errors remain visible;
intentional closure during transport retry is not itself a source error.
Keep response bounds, JSON validation, output installation, retry
classification/delay, cancellation, and application attempt reporting intact.
4. Remove obsolete producer-only helpers rather than retaining two upload paths.
Adapt tests tied to producer goroutines to assert streaming and resource
release, which are the actual contracts.
5. Add focused body tests: original/replayed bytes parse identically; replay
after partial consumption starts from the beginning; simultaneous bodies use
independent handles; source failures propagate; Close/cancel releases a
blocked reader without deadlock. Use synthetic audio and gated readers.
Verify: `go test ./internal/adapters/whisperx` and
`go test -race ./internal/adapters/whisperx`.
Done when replay is available without whole-file buffering or producer-lifetime
ambiguity, and all existing HTTP behavior tests still pass.
## Stage 6: Verify Replay Through the HTTP Adapter Boundary
Status: Complete.
Completion marker (2026-09-22): Added a deterministic transport regression
that abandons a partially consumed original request, obtains and parses a fresh
multipart replay from `GetBody`, and installs the returned transcript within one
application attempt. Covered replay open/read failures, early responses,
parent cancellation during replay, and disposal of every opened source, while
proving an abandoned original does not poison a valid replay. Added a real HTTP
retry-exhaustion integration that durably retains an earlier speaker and resumes
only the uncommitted speakers. This controlled boundary verifies Narratio's
replay contract, not recovery from every real proxy protocol failure. Verified
with `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`,
`go test -race ./internal/adapters/whisperx ./internal/app ./internal/stage`,
ten repeated race-enabled adapter runs, and `git diff --check`.
Preparation: Existing injectable HTTPClient/RoundTripper seam and local-server
tests in `internal/adapters/whisperx/http_test.go`; app WhisperX wiring tests.
1. Add a deterministic transport-boundary regression: consume part of the
original request, close it as a transport would, call GetBody, consume and
parse the replay using the request Content-Type, then return valid JSON.
Verify exact audio/language, installed output, and one application attempt.
This must fail before replay support; merely checking non-nil GetBody is
insufficient. Do not assert Go's internal retry count or require a flaky
handcrafted HTTP/2 peer in the default suite.
2. Cover replay-open/read failure, early response, parent cancellation during
replay, and disposal of all body handles. Reuse Stage 5 body coverage rather
than repeat every low-level case. Prove an intentionally abandoned original
body does not turn a valid replay response into failure.
3. Retain local HTTP tests for application retries, non-retryable responses,
response limits, invalid JSON, and bounded streaming; exercise that an
exhausted HTTP failure still leaves earlier speaker checkpoints reusable
using the existing stage/app seams.
4. Record the verification boundary honestly: controlled transport replay proves
Narratio's body contract, not recovery from every real proxy PROTOCOL_ERROR.
Verify: `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`
and race tests for changed packages.
Done when the replay defect is protected at the adapter boundary and the
application retry/checkpoint behavior remains intact.
## Stage 7: Document and Validate the Completed Feature
Status: Complete.
Completion marker (2026-09-22): Updated the canonical transcribe, merge,
manifest, WhisperX, operations, and troubleshooting documentation for durable
speaker reconciliation, serialized checkpoints, aggregate gating, durable
force reset, legacy upgrade and downgrade limits, replayable streaming bodies,
resource ownership, and the absence of an exactly-once remote guarantee.
Reviewed the completed implementation against the feature roadmap and
implementation-discipline guidance. Verified with `go test ./...`,
`go test -race ./...`, `go vet ./...`, `go build ./...`,
`go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check`.
1. Update canonical transcribe and manifest internals for the collection,
reconciliation, durable-reset force behavior, serialized saves, and aggregate
gate. Remove obsolete all-speakers-before-canonical-write descriptions.
2. Update the WhisperX contract for replayable bounded streaming, application
attempts versus transport replay, resource ownership, and the lack of an
exactly-once server guarantee. Do not claim the remote protocol issue is fixed.
3. Update operations/troubleshooting with ordinary resume versus force, durable
partial success, old-manifest one-time retranscription on selecting transcribe,
downgrade limitations, and private-model-change guidance. No new CLI flags
or promises of automatic historical-file recovery. Keep detailed wire and
storage contracts in their canonical owners.
4. Review changes against the feature roadmap and the implementation-discipline
guidance. Do not elevate selected storage encoding, internal ordering, test
synchronization, or exact error wording into new product guarantees.
5. Run the full repository gates below; document results and update completion
statuses only after they pass. Leave release preparation for a separate task.
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go test ./internal/doccheck
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
git diff --check
```
## Open Questions
None requiring user input. The agreed roadmap supplies the product decisions;
this plan selects a completed-only manifest collection, conservative old-record
handling, and pull-based replay as implementation choices. No stronger remote
reliability guarantee, new operator restriction, or generic orchestration
surface is proposed.

View File

@@ -0,0 +1,151 @@
# Notarius Configuration Profile Selection
Status: Implemented.
## Goal and Authority
Implement the consumer behavior in the
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
configuration overlay explicitly select a Notarius configuration overlay.
Operators can select testing once in Narratio while each application retains
ownership of its own model choices and other settings.
This roadmap describes the target state and required change areas, not an
implementation sequence; the [implementation plan](implementation.md) owns
the ordered work and implementation decisions. The feature request is the scope
authority; the mechanisms suggested below do not establish additional
user-facing contracts.
## Underlying Issue
Narratio can select its own testing profile, but extraction currently invokes
Notarius without a configuration-profile selector. Consequently, a Narratio
testing invocation cannot explicitly select the independently deployed Notarius
testing overlay through pipeline configuration.
Adding individual downstream model or tuning fields would duplicate Notarius's
configuration responsibilities. Passing its existing profile selector provides
the requested flexibility without that duplication.
## Targeted End State
### Explicit Configuration Composition
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
name, usable in Narratio's base configuration and ordinary overlays. For
example, Narratio's `profiles/testing.yml` may contain:
```yaml
notarius:
config_profile: testing
```
Selecting Narratio's `--profile testing` then selects that explicit downstream
profile whenever extraction executes. The two profile names need not match.
Narratio does not automatically forward its own profile name.
When supplied, the selection is passed through Notarius's `--config-profile`
argument. Omitted or null values add no profile-selection flag and preserve
existing downstream selection behavior. Notarius's selector semantics remain
authoritative, including empty and `default` selections for the composed base
configuration. Narratio does not introduce a stricter profile-name policy.
This field selects a ConfigKit configuration overlay, not a PromptKit profile
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
selection. Notarius resolves overlays relative to its own configuration root;
Narratio neither resolves those files against its own root nor rewrites them.
### Execution, Reuse, and Failure
A changed downstream profile selection must not silently reuse Narratio's
completed extraction from the previous selection. Existing dependent-stage
invalidation applies when extraction changes; the feature does not change the
pipeline order or transcript-generation behavior.
Notarius continues to own downstream overlay discovery, composition, validation,
and cache compatibility. A missing or invalid selected overlay must fail
extraction through the existing subprocess error path, without silently falling
back to production or reporting successful extraction.
Existing resume and chunk-cache controls, including force precedence, remain
in place. Profile selection does not itself mean forced regeneration or cache
deletion. Narratio's decision to execute extraction and Notarius's decision to
reuse compatible work remain separate responsibilities.
Changes to private Notarius configuration behind an unchanged selector retain
the existing operator responsibility to force extraction when needed. There is
no new downstream file inspection or content-tracking requirement.
Credential forwarding remains independently configured through
`notarius.llm_api_key`. A Narratio testing overlay may select a different
credential reference alongside the downstream profile; no new credential
discovery or provider-name mapping is introduced.
## Required Change Areas
### Configuration and Inspection
Extend the typed Notarius configuration and its existing safe effective-YAML
projection to carry the optional field. Preserve the ordinary configuration
composition behavior for base values, overlay overrides, and null values.
Configuration inspection must reflect the selected field without exposing
credential values. No replacement configuration loader or downstream preflight
loader is needed.
### Stage and Adapter Boundary
Carry the selection from resolved configuration through the extraction request
to the subprocess adapter. Construct the optional CLI flag using the existing
argument-vector mechanism, alongside the current configuration path, references,
credential forwarding, and cache controls. Retain normal Notarius error handling.
### Extraction Reuse Evidence
Account for the downstream selection in the existing extraction reuse evidence.
The current semantic-configuration and invocation fingerprints are the relevant
change points. Keep their treatment of selection consistent so that a profile
change cannot bypass the required rerun and downstream invalidation.
This roadmap does not require a new manifest schema, separate profile-indexed
workspaces, historical per-profile result retention, or an upgrade-wide forced
rerun.
### Verification and Documentation
Use focused behavioral coverage for configuration composition and inspection,
explicit argument forwarding and omission, changed-selection reuse decisions,
and ordinary subprocess failure handling. Reuse existing test coverage where
it already protects unchanged credentials, force behavior, and invalidation;
do not duplicate Notarius's complete profile-validation or cache test suites.
When implemented, update the canonical
[configuration reference](../config.md),
[Notarius integration contract](../integrations/notarius.md),
[extract internals](../internal/stage-extract.md), and relevant maintained
examples. The development workflow in [Operations](../operations.md) should
explain explicit profile composition and the existing need to force extraction
after private downstream edits. Clarify that profile selection does not create
an isolated testing workspace or separate current extraction state.
## Compatibility and Scope Boundaries
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
required. Document the capability needed when the option is used, without adding
an unconditional runtime version gate for existing unconfigured deployments.
This feature adds no Narratio CLI selector, automatic profile propagation,
Notarius LLM-profile selector, downstream configuration rewriting, new
credential mechanism, or generalized recursive orchestration. Existing output
and receipt contracts remain unchanged.
The consumer's Ansible role and suite updates are follow-up work outside this
repository. They can deploy downstream overlays and populate this field after
the interface is released.
## Scope Review
No stronger requirements or new invariants are proposed beyond the feature
request and existing repository contracts. In particular, minimizing one-time
upgrade invalidation and avoiding reruns between equivalent base selectors are
implementation considerations, not additional acceptance guarantees imposed
by this roadmap. No user decisions are currently required before planning.

104
docs/roadmap/remediation.md Normal file
View File

@@ -0,0 +1,104 @@
# Transcription Checkpoint Review Remediation
Status: Complete.
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
upload-replay roadmap and the original user request. No production changes were
made during review. The two failures below were reproduced with temporary tests
using the existing stage fixtures; those temporary tests were removed afterward.
## 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
Priority: P2. Status: Complete.
Implemented read-only aggregate identity/evidence checks before resume skips.
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
repair without HTTP requests, downstream invalidation, and exclusion of removed
speakers from merge without deleting their files. Equivalent path forms and
output ordering remain reusable. Focused app/stage tests and documentation
checks pass.
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
consumed there.
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
only establishes that the remaining speakers need no transcription; it does
not establish that the stored speaker collection and aggregate outputs match
the current input set. Removing a speaker while the remaining inputs are
unchanged produces an empty work list, so a succeeded record can be skipped
without pruning the removed speaker. Its old aggregate output can then remain
available to merge. Likewise, missing/inconsistent aggregate output references
are not repaired when all individual checkpoints remain valid. A prepare rerun
may already stale transcribe in some workflows, but the resume validator cannot
rely on that to satisfy its own current-input/output checks.
Reproduction: create complete alice/bob checkpoint records and aggregate
outputs with the existing fixture, mark the stage succeeded, remove bob from
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
despite still retaining bob in the stored collection and aggregate outputs.
Required change:
- Distinguish no HTTP work from no reconciliation work. Validate that the
committed speaker set and aggregate transcript references represent the
reconciled current set, including relevant identity/evidence, before skipping.
- When only collection/output repair is needed, return non-resumable and let
normal execution checkpoint the pruned collection and rebuild aggregate
outputs without retranscribing reusable speakers. Preserve existing dependent
invalidation; do not force otherwise-current speaker jobs.
- Keep resume inspection read-only. Do not fix this by deleting incidental
transcript files or changing stage selection rules.
Verification: add removed-only input coverage (not just removal combined with
an added speaker), plus incomplete/inconsistent aggregate output coverage.
Exercise the runner skip path and assert zero additional WhisperX requests,
correct rebuilt outputs, and no removed speaker reaching merge.
## 2. Remove the Unrequested Positive Audio-Size Requirement
Priority: P2. Status: Complete.
Audio evidence now permits zero size while still rejecting negative values.
Regression tests verify zero-size manifest round trips, successful service
results being checkpointed and reused, and service rejection being propagated
without creating a completion record. Nonempty valid-JSON output validation is
unchanged. Both new success regressions failed before the fix.
Location: `internal/manifest/transcribe_state.go:95`.
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
a regular `.flac` file and delegated transcription acceptance to WhisperX;
neither the user request nor the roadmap introduced minimum audio size or a new
audio-content validation policy. Zero-byte content still has a valid checksum
and can be represented by the checkpoint mechanism. The current check runs
after a successful response and output materialization, causing the entire
stage to fail instead of retaining the completed work.
Reproduction: prepare one speaker, replace its prepared audio with an empty
regular file, and use the existing successful fake WhisperX client. Stage
execution fails at checkpoint validation with `audio_size must be positive`.
This demonstrates the extra Narratio restriction; it does not assert that a
particular production WhisperX deployment accepts empty audio.
Required change:
- Permit zero audio size and reject negative sizes. Keep byte hashing and
size equality checks. Let the downstream service decide whether the input
can be transcribed, as before.
- Do not remove the nonempty valid-JSON output requirement, which has a separate
existing justification. Do not add a new early empty-audio rejection instead.
- Update any new tests/documentation that promote strictly positive input size
to a contract.
Verification: cover a zero-size record round-trip and successful adapter result
being checkpointed/reused for that input; retain negative-size rejection and
ordinary downstream-failure tests.
## Verification After Remediation
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
changed-package race tests, `go test ./...`, `go test -race ./...`,
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check` all passed. Existing replay and cancellation regression
tests remain intact; the HTTP adapter required no changes.

View File

@@ -0,0 +1,183 @@
# Resumable Speaker Transcription and Replayable Uploads
Status: Implemented.
## Goal and User Intent
Preserve completed speaker transcriptions when another speaker fails, so an
ordinary run or resume does not repeat expensive successful work. Separately,
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
recoverable connection or stream failure.
The motivating incident exhausted four transcription attempts for one speaker
after three other speakers had completed. The final error reported an HTTP/2
peer `PROTOCOL_ERROR` and an inability to replay the request body without
`Request.GetBody`.
These are related reliability improvements, not a promise to eliminate all
service failures. This document owns the target state and scope; the
[implementation plan](implementation.md) owns ordered changes and concrete
storage and replay decisions.
## Pre-implementation Limitations
The [transcribe stage](../internal/stage-transcribe.md) writes successful
responses into the invocation's run directory. It installs canonical transcripts
and returns its per-file metadata only after every speaker succeeds. A failure
therefore leaves no durable per-speaker progress that ordinary resume can use,
even though some completed JSON files may remain on disk.
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
the application level, reopening audio for each attempt. Its multipart request
body is streamed through a pipe and has no replay callback. Transport-level
recovery within an attempt cannot reconstruct a consumed body. Providing that
capability addresses the replay limitation, not the underlying peer protocol
error; persistent failures can still require service or proxy investigation.
## Targeted End State
### Durable Per-Speaker Completion
Each successfully transcribed speaker file is validated, saved, and recorded
durably before that work is treated as reusable. The session manifest remains
the progress authority. Follow the existing analyze-stage checkpoint pattern
without turning speakers into pipeline stages or introducing a general task
engine.
Ordinary execution reconciles the current prepared audio set with recorded
speaker results. It reuses valid completed work and schedules failed,
unattempted, missing, or obsolete results. A result is not reusable merely
because a matching filename exists. Evidence binds it to the speaker/audio
identity, audio content, Narratio-visible recognition settings, and the saved
transcript's integrity.
The relevant recognition settings remain the WhisperX service URL and language.
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
remain operational settings rather than reasons to discard completed results.
Changes to private service models or configuration remain an operator reason
to force transcription; Narratio does not inspect the service's private state.
Changes to individual audio inputs require replacement of affected results,
not automatic retranscription of unrelated unchanged speakers. New speakers
need transcription; removed speakers do not remain in the current aggregate
output set. Records must describe the inputs actually used, not grant reuse to
old bytes under a changed identity.
### Aggregate Completion, Failure, and Force
The overall stage succeeds only when every currently required speaker has a
validated, durably recorded result. Incomplete per-speaker progress does not
make the aggregate stage successful or authorize downstream consumption of an
incomplete collection. Existing dependent-stage invalidation remains in effect.
Retain bounded concurrency and the current fail-fast behavior: a speaker
failure cancels outstanding work. Results already checkpointed remain reusable
after failure, cancellation, or process interruption. A later ordinary run or
resume retries the remaining work; it does not need a new CLI command.
Checkpoint writes must be serialized so concurrently finishing speakers cannot
overwrite each other's progress. A failed persistence operation must not
advertise an unrecorded result as complete. Previously committed progress remains
the recovery authority, and checkpoint errors must be surfaced rather than
reported as successful transcription.
`--force` on transcribe regenerates all currently required speakers instead of
reusing their prior completions. Successful replacements are checkpointed too,
so an ordinary retry after a failed forced invocation can retain the newly
completed work. Old results for speakers not successfully regenerated must not
satisfy that unfinished forced replacement on the next ordinary retry.
This deliberately relaxes the current all-speakers-before-canonical-installation
implementation: individual durable results may exist while the stage remains
failed or incomplete. The all-speakers-before-stage-success boundary remains.
### Replayable WhisperX Requests
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
same request. Replay reopens the audio and reproduces the multipart boundary,
filename, fields, and framing used by the original request. Independent bodies
must not share a consumed reader or mutable read position.
Preserve bounded-memory streaming rather than buffering entire audio files in
memory. Correctly close every original and replay body and its audio resources;
preserve source-read errors and cancellation, including partial uploads and
early server responses. Successful response handling must not accidentally wait
on, or report errors solely from, an abandoned body instead of the replay that
produced the response.
Keep the existing application-level retry count, delay, classification, and
per-attempt timeout semantics. Transport replays occur within an application
attempt and its context; they are not additional Narratio-configured attempts.
Do not add another custom retry loop, silently increase retries, or disable
HTTP/2 globally as a substitute for replay support.
The adapter still validates and installs only successful JSON responses.
Request replay does not provide exactly-once server execution: a lost response
can cause repeated transcription work, as can existing application retries.
No new service-side idempotency protocol is required by this feature.
## Required Change Areas
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
evidence, with safe checkpoint updates and preservation through runner failure
handling. Review manifest validation, copying, serialization, and existing
publish/restore handling wherever the new records pass through those paths.
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
checkpoint validated completions, rebuild aggregate outputs from the current
set, and account for partial progress in ordinary and forced execution.
Review successful-stage reuse as well as failed-stage retries so stale audio
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
directory surviving cleanup. Do not introduce a second authoritative ledger
or infer completion by scanning abandoned runs.
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
an upstream service change.
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
checkpoint failure; changed/missing inputs or outputs; force followed by an
ordinary retry; concurrent successful completions; and aggregate downstream
gating. Extend existing fixtures rather than duplicating analyze's complete
test suite. Exercise actual body reconstruction after partial consumption,
multipart equivalence, cancellation/resource cleanup, and existing bounded
streaming and retry behavior—not just the presence of a callback. Default
tests remain offline and use local/fake HTTP boundaries, never paid services.
- **Documentation:** Update the canonical
[stage guide](../internal/stage-transcribe.md),
[manifest guide](../internal/manifest.md),
[WhisperX contract](../integrations/whisperx.md),
[operations guide](../operations.md), and
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
partial completion, ordinary retry versus force, and the distinction between
request replay limitations and persistent peer protocol failures.
## Compatibility and Scope Boundaries
Existing speaker identities, raw transcript formats, and the fixed pipeline
order remain unchanged. Existing manifests remain readable, but must not be
treated as containing per-speaker evidence they never recorded. Recovery relies
on verified recorded evidence, not guessed associations between old transcripts
and current audio. No general legacy-migration framework is part of the feature.
Automatic adoption of unrecorded outputs from previously failed runs is outside
this feature. Such files may be recoverable after verifying their inputs, but
their presence alone does not prove completion. No recovery/import command,
speaker-selection CLI, historical invocation ledger, new service configuration
discovery, or broader orchestration refactor is included.
The feature does not diagnose the remote service/proxy's underlying HTTP/2
problem, guarantee recovery from persistent protocol errors, or change the
policy for continuing unrelated work after failure.
## Completion Criteria
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
completed, ordinary resume retains A and B, retries C, and completes D. No
downstream stage receives an incomplete successful transcription collection.
Force replaces all four, while an ordinary retry can reuse checkpoints from
that forced replacement attempt.
The HTTP transport can reconstruct the upload after consuming the original
body, without whole-file memory buffering, lost source errors, or stranded
upload resources. A persistent peer error still fails normally after the
configured application attempts, without discarding checkpointed speakers.

View File

@@ -24,6 +24,8 @@ Safe fix:
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`. - pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
Relevant reference: [Configuration discovery](./config.md#discovery-and-selection).
## Session template placeholders rejected ## Session template placeholders rejected
Symptom: Symptom:
@@ -44,6 +46,8 @@ Safe fix:
- generate concrete session YAML with `narratio session init`. - generate concrete session YAML with `narratio session init`.
Relevant reference: [Operations: Session Initialization](./operations.md#session-initialization).
## Strict decode or schema validation failure ## Strict decode or schema validation failure
Symptom: Symptom:
@@ -52,7 +56,8 @@ Symptom:
Likely cause: Likely cause:
- stale field name, typo, invalid enum, or invalid duration/path format. - stale field name, typo, invalid enum, invalid duration/path format, or an
unsupported local manifest schema/state marker.
Diagnostics: Diagnostics:
@@ -62,7 +67,70 @@ narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /pa
Safe fix: Safe fix:
- align config with [docs/config.md](./config.md) and maintained files under `examples/`. - align config with [Configuration](./config.md) and the
[maintained examples](../examples/README.md).
- for a manifest schema error, move the unsupported local state aside and
regenerate it; restore a verified current committed snapshot or republish
healthy regenerated state when appropriate. Do not add version or stage
markers by hand.
- if an older manifest is rejected for its retired cleanup-obligation field,
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
remote commit, move aside or regenerate the incompatible manifest, and use
the cleanup stage. Do not treat manual JSON editing as the normal migration
path.
Relevant reference: [Configuration](./config.md).
## Party schema version rejected
Symptom:
- configuration loading reports that `party.schema_version` is required or
unsupported.
Likely cause:
- the campaign `party.yml` is an unversioned legacy document, or names a
schema other than `narratio.party.v1`.
Safe fix:
- replace the file with one exact, single-document
[`narratio.party.v1` party](./integrations/party.md); do not add a version
marker to an otherwise incompatible legacy structure. The versioned party is
the only configured roster authority.
Relevant reference: [Canonical Party Input](./integrations/party.md).
## Unexpected imported or profile value
Symptom:
- an effective configuration value differs from the root file, or a duplicate
ownership/configuration error is hard to locate.
Diagnostics:
```bash
narratio config sources --config /path/pipeline.yml --profile testing
```
Add `--campaign` or `--campaign-file` when the pipeline has party-driven
artifact families. The output identifies each effective logical field's root,
import, profile, default, campaign, party, or family source without printing
the field value or credential contents.
Safe fix:
- move a duplicated base field so it has one owner;
- correct the selected profile or its overlay; or
- correct the campaign party/family declaration that owns generated values.
To review what would actually change before switching profiles, run `config
diff` with the same pipeline and campaign selectors. It compares normalized
effective values rather than YAML formatting or source-file layout.
Relevant reference: [Configuration inspection](./config.md#read-only-effective-pipeline-inspection).
## Audio mode conflict ## Audio mode conflict
@@ -74,10 +142,18 @@ Likely cause:
- configured both local and S3 session audio inputs. - configured both local and S3 session audio inputs.
Diagnostics:
```bash
narratio session validate 2026-04-04
```
Safe fix: Safe fix:
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both. - use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
Relevant reference: [Session configuration](./config.md#session).
## `--artifacts` selection error ## `--artifacts` selection error
Symptom: Symptom:
@@ -90,10 +166,569 @@ Likely causes:
- empty list entry (for example trailing comma); - empty list entry (for example trailing comma);
- `run-stage` used with non-`analyze`/`publish` target. - `run-stage` used with non-`analyze`/`publish` target.
Diagnostics:
```bash
narratio session artifacts 2026-04-04
```
Safe fix: Safe fix:
- provide only configured keys and use `--artifacts` with supported commands/stages. - provide only configured keys and use `--artifacts` with supported commands/stages.
Relevant reference: [CLI artifact selection](./cli.md).
## Bounded run prerequisite is unusable
Symptom:
- `run` or `session plan` reports that a prerequisite stage is absent or has a
pending, running, failed, stale, or interrupted status before the selected
start.
Likely cause:
- `--from` excludes upstream work that has not reached the terminal
`succeeded` or `skipped` state in the session manifest.
Diagnostics:
```bash
narratio session status 2026-04-04
narratio session plan 2026-04-04 --from render --through analyze
```
Safe fix:
- widen the bounded range to include the first reported stage, or recover that
stage explicitly with `run-stage` before retrying. The failed check does not
create a run-local directory or modify the manifest. Narratio does not
resume-validate excluded prefix stages, and stages after `--through` are not
prerequisites.
If prerequisite statuses are terminal but a selected stage reports a missing,
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
that owns it; do not edit the manifest to bypass the selected stage's concrete
input validation.
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
## Transcription retries speakers that previously finished
Symptom:
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
finish earlier; or
- an older successful session retranscribes all speakers once after upgrade.
Likely causes:
- the earlier response or canonical file was not followed by a successful
speaker manifest checkpoint;
- the prepared audio bytes/path, WhisperX URL, or language changed;
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
no longer matches its recorded size/checksum;
- the previous invocation used `--force`, which durably cleared pre-force
speaker authority; or
- the manifest has only legacy aggregate transcribe state and cannot bind old
outputs to their source audio.
Safe fix:
- inspect the reported speaker and filesystem error, repair the prepared input
or canonical-output path when needed, then retry transcribe normally;
- allow a legacy aggregate-only record to regenerate once; and
- do not edit the manifest or copy an old JSON file into place as a substitute
for a successful checkpoint.
```bash
narratio run-stage transcribe 2026-04-04
```
Valid earlier speaker checkpoints remain reusable even while aggregate
transcribe status is failed or interrupted. Merge remains gated until
transcribe reaches aggregate success with the complete current set.
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
## WhisperX model changed but transcription was reused
Symptom:
- WhisperX model weights, prompts, or private server configuration changed at
the same configured URL, but Narratio still considers speaker transcripts
current.
Likely cause:
- Narratio can fingerprint its configured URL and language, not private state
loaded behind the service boundary.
Safe fix:
- explicitly force transcribe to establish a durable reset, then use ordinary
retry if the replacement attempt is interrupted.
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Transport replay and application retries do not provide an exactly-once remote
execution guarantee. Investigate server-side failures or duplicate work using
WhisperX diagnostics; do not infer completion from a server log without the
local speaker checkpoint.
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
[Transcribe internals](./internal/stage-transcribe.md).
## Transcription manifest checkpoint failed
Symptom:
- transcribe reports that a speaker completed but the session manifest could
not be saved.
Likely causes:
- the workspace is unwritable or out of space;
- the manifest path or parent directory is unsafe or the wrong type; or
- a transient filesystem error interrupted atomic replacement.
Safe fix:
- correct the reported filesystem or permission problem and retry transcribe
normally. Narratio reconciles from the last readable collection and reuses
only successfully checkpointed speakers.
A canonical raw transcript left by the failed checkpoint is incidental and
does not become reusable authority. Preserve the manifest and diagnostics if
the failure repeats; do not advertise the file by editing manifest evidence.
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
and [Transcribe internals](./internal/stage-transcribe.md).
## Notarius credential reference unavailable
Symptom:
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
name for `pipeline.notarius.llm_api_key`, including during inspection or a
run that would not execute extraction.
Likely causes:
- the named environment variable is absent or empty;
- the selected secrets directory lacks a file with that exact name;
- a process environment variable exists but is empty, so bootstrap correctly
leaves it in place; or
- the Narratio reference name does not match `api_key_env` in the selected
Notarius/PromptKit profile.
Safe fix:
- set the exact nonempty environment variable or provide an equivalently named
file through the conventional or explicit secrets directory;
- remove or set the field to `null` when that deployment needs no credential;
and
- use `config show`, `config sources`, and `config diff` to inspect the public
reference name and ownership without exposing its value.
Narratio resolves selected credential references during ordinary pipeline
loading, before stage selection or resume. Credential changes alone do not
invalidate a completed extraction, but the current reference must still resolve
for the command to proceed.
Relevant references: [Secrets](./config.md#secrets-handling) and
[Notarius credential](./config.md#notarius-credential).
## Notarius executable missing
Symptom:
- extraction fails while resolving or starting the Notarius executable.
Likely causes:
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
- a configured executable path is wrong.
Safe fix:
- install a compatible Notarius release or correct the binary setting, then
rerun extraction.
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
and [Notarius integration](./integrations/notarius.md).
## Notarius exits nonzero
Symptom:
- extraction reports a Notarius exit error instead of a receipt.
Diagnostics:
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
the receipt and is not merged with diagnostics.
Safe fix:
- correct the reported Notarius pipeline, input, provider, or configuration
failure and rerun extraction. Do not edit a staged output bundle into place.
After a failed replacement, an older immutable bundle may still exist even
though the current session manifest has no successful extraction payload. This
is expected audit state, not a signal to relink the old bundle manually.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Prepared Notarius reference missing or inconsistent
Symptom:
- extraction or resume validation reports that a configured reference source is
unavailable, unsafe, empty, or checksum-inconsistent and recommends
`prepare --force`.
Likely causes:
- `prepare` has not run since the campaign/session stable input changed;
- the configured source file is missing;
- a prepared `inputs/` file or its manifest record was modified independently;
- a spell-catalog binding exists without an effective `spell_catalog_file`.
Diagnostics:
```bash
narratio session status 2026-04-04
narratio session validate 2026-04-04
```
Safe fix:
- correct the campaign/session input path, then refresh canonical prepared
evidence before extraction:
```bash
narratio run-stage prepare 2026-04-04 --force
```
Do not point Notarius directly at the original source path or edit the manifest
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Notarius reference snapshot cache conflict
Symptom:
- extraction reports that a path beneath
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
non-regular or unsafe entry.
Safe fix:
- stop commands for the affected session and inspect the exact reported path;
- investigate unexpected links or special files as possible tampering; and
- remove only the reported conflicting leaf or digest directory after
confirming it belongs to the reconstructible reference cache.
A missing snapshot or corrupt regular snapshot is reconstructed automatically
from currently verified prepared input. Do not recursively delete the session
root, edit the manifest checksum, or remove cache entries while extraction is
active.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Notarius resume executes work instead of reusing it
Symptom:
- Narratio executes `extract`, and Notarius makes provider calls despite
`pipeline.notarius.resume: true`.
Likely causes:
- checkpoint recording is disabled in the Notarius configuration;
- Notarius is using a nonpersistent or different checkpoint root;
- transcript/reference bytes, reference paths, pipeline/profile content,
local-backend endpoint, selected lanes, or another checkpoint identity input
changed; or
- Narratio extraction was forced, which deliberately omits `--resume`.
Safe fix:
- validate the deployed Notarius configuration and configure explicit
persistent chunk-plan and checkpoint roots;
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
and
- retry without force only when reuse is intended.
Remember that Narratio skipping a completed `extract` stage is separate from
Notarius reusing work inside an extraction that does execute. Narratio does not
read the Notarius configuration to infer checkpoint enablement.
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
and [Notarius integration](./integrations/notarius.md).
## Notarius reference selector or generated-handoff collision
Symptom:
- Notarius exits nonzero with an undeclared reference-slot, incompatible media,
or external/generated reference collision error.
Likely causes:
- a selector does not identify a slot declared by the selected Notarius target;
- a prepared file does not satisfy that slot's Notarius media contract; or
- a CLI binding attempts to replace a same-run generated D&D handoff.
Safe fix:
- compare external bindings with the selected Notarius pipeline's canonical
consumer documentation;
- keep only campaign-owned external slots on the CLI; and
- leave registry, scene, combat, and occurrence handoffs to Notarius pipeline
composition.
Narratio validates selector structure and prepared evidence, while Notarius
owns slot declarations, media compatibility, and generated-handoff conflicts.
Relevant reference: [Notarius integration](./integrations/notarius.md).
## Notarius receipt or index incompatible
Symptom:
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
lane descriptor, or payload path even though Notarius exited successfully.
Likely causes:
- Narratio and Notarius versions disagree on their consumer contract;
- the configured pipeline or lane constraints are stale;
- output paths escape the bundle or traverse symlinks.
Safe fix:
- compare installed Notarius output with the canonical Notarius contracts,
including receipt `index_file: index.json` and index management names
`manifest.json`, `rejected.json`, `warnings.json`, and `diagnostics.json`; align
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
checks.
Relevant reference: [Notarius integration](./integrations/notarius.md).
## Required Notarius lane rejected or missing
Symptom:
- extraction fails because a configured lane is rejected, missing, duplicated,
or incompatible, including after a zero exit.
Safe fix:
- inspect the Notarius diagnostic log and bundle rejection/warning information;
- correct the Notarius module or the exact declared lane contract;
- remove an output declaration only if downstream consumers genuinely no longer
require that source, then rerun extraction.
Every configured output is required. Narratio does not promote a partial result.
## Extraction resume invalidated
Symptom:
- a previously successful extraction runs again during ordinary continuation.
Likely causes:
- the pipeline ID or configured output contracts changed;
- a configured prepared reference selector, source, path, checksum, or byte
size changed;
- the direct trimmed transcript identity or bytes changed; or
- the durable bundle, index, lane set, provenance, regular-file status, or
checksum no longer validates.
Safe fix:
- allow the automatic rerun after verifying the current configuration. Treat
an unsafe path or bundle symlink error as filesystem corruption or tampering
and investigate it rather than replacing files manually.
Executable, config path, timeout, working directory, and Notarius credential
changes are operational and do not by themselves invalidate reusable extraction
evidence. A newly selected credential must nevertheless resolve during command
configuration loading.
## Notarius transitive configuration changed
Symptom:
- Notarius profiles, prompts, modules, imported files, or references changed,
but Narratio still considers the previous extraction resumable.
Safe fix:
```bash
narratio run-stage extract 2026-04-04 --force
```
Narratio fingerprints its invocation contract and prepared Narratio reference
identities, not the contents of other transitive Notarius inputs. Always force
extraction after changing those external inputs; downstream
successful stages are then marked stale normally.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Analysis artifact evidence is not current
Symptom:
- ordinary continuation or `session plan` schedules one or more configured
artifacts even though a canonical output file exists; or
- publish reports a configured artifact source unavailable.
Likely causes:
- the per-artifact record is stale, missing, failed, unselected, or malformed;
- a configured prompt/profile, dependency, input identity, output path, or
effective variable changed; or
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
longer matches its manifest evidence.
Diagnostics:
```bash
narratio session status 2026-04-04
narratio session artifacts 2026-04-04
narratio session plan 2026-04-04 --from analyze --through analyze
```
Safe fix:
- investigate unexpected path or checksum changes as possible tampering;
- otherwise let the selected analyze work rerun, or explicitly regenerate only
the affected targets; and
- never edit the fingerprint/checksum in the manifest or copy an old file into
the canonical path as a substitute for current evidence.
```bash
narratio analyze 2026-04-04 --artifacts session_recap
```
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
## Analysis manifest checkpoint failed
Symptom:
- analyze reports that it could not save the session manifest after completing
an artifact.
Likely causes:
- the workspace is unwritable or out of space;
- the manifest path or its parent directory has unsafe permissions or type; or
- a transient local filesystem error interrupted atomic manifest replacement.
Safe fix:
- correct the reported filesystem or permission problem;
- inspect `session artifacts` to identify the last durable current records; and
- rerun analyze normally. Narratio reuses earlier successful checkpoints and
retries the failed or non-current artifact and its required dependents.
Do not treat a newly created canonical artifact file as current when its
checkpoint failed, and do not edit the manifest to advertise it. If the error
recurs, preserve the manifest and run-local diagnostics before repairing the
workspace.
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
and [Manifest Internals](./internal/manifest.md#persistence-semantics).
## Built-in transcript or bounds source is unavailable
Symptom:
- analyze, extract, render, or publish says a built-in source is unavailable
even though a file exists at its usual session path.
Likely cause:
- the producing stage has not succeeded, its output lacks the exact registered
source, run ID, or checksum, its evidence is duplicated or stale, or the
durable bytes no longer match the manifest checksum.
Safe fix:
- do not copy or rename the file to make it discoverable. Rerun the producing
stage to regenerate both the bytes and current manifest evidence, then retry
the consumer.
## An older manifest schema is rejected
Symptom:
- Narratio rejects a local session manifest because `schema_version` is missing
or unsupported; or
- restore rejects a committed remote session manifest for the same reason.
Likely causes:
- the manifest predates the supported canonical state contract. Narratio
rejects it at the schema boundary rather than interpreting aggregate output
lists or incidental files as current authority; or
- an older strict-decoding Narratio binary is reading a current manifest that
contains additive transcribe speaker state it does not recognize.
Safe fix:
- for unsupported local state, move the old manifest aside and regenerate, or
restore a verified current committed snapshot when one is authoritative;
- for unsupported committed remote state, regenerate healthy local state as
needed and republish the session; and
- for a downgrade rejection, return to the current binary or restore a verified
pre-upgrade backup. Do not delete only the speaker fields by hand.
```bash
narratio run 2026-04-04
narratio publish 2026-04-04
```
After current records exist, inspect them and publish explicitly. Do not copy
old files into canonical paths merely to influence selection; availability is
manifest-owned.
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
## Scriptorium private input changed without a rerun
Symptom:
- a prompt, profile, imported configuration file, executable, or other input
loaded privately by Scriptorium changed, but Narratio still considers an
artifact current.
Likely cause:
- analysis fingerprints cover Narratio-observable semantic identities, not
executable contents or arbitrary files and transitive configuration that
Scriptorium loads behind its configured paths and identifiers.
Safe fix:
- explicitly force the affected target after changing an unobserved private
input. Force applies to explicit targets; current prerequisites remain
reusable unless selected themselves.
```bash
narratio analyze 2026-04-04 --artifacts session_recap
```
Relevant reference: [Analyze Internals](./internal/stage-analyze.md#invariants).
## Previous-session artifact input missing ## Previous-session artifact input missing
Symptom: Symptom:
@@ -103,7 +738,9 @@ Symptom:
Likely causes: Likely causes:
- missing `session.previous_session_id`; - missing `session.previous_session_id`;
- previous artifact not restored/published for source session. - the selected previous-session commit does not declare the required canonical
source as a `published_output`; or
- the declared published object is unavailable or fails identity validation.
Diagnostics: Diagnostics:
@@ -124,6 +761,8 @@ or rerun prepare after correcting session config:
narratio run-stage prepare 2026-04-04 --force narratio run-stage prepare 2026-04-04 --force
``` ```
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
## Session lock conflict (`.lock`) ## Session lock conflict (`.lock`)
Symptom: Symptom:
@@ -133,7 +772,7 @@ Symptom:
Likely causes: Likely causes:
- another process is running for the same session; - another process is running for the same session;
- stale lock left by interrupted process. - a process still holds the operating-system lock while it is shutting down.
Diagnostics: Diagnostics:
@@ -145,7 +784,10 @@ ps aux | grep narratio
Safe fix: Safe fix:
- wait for active process completion; - wait for active process completion;
- remove stale lock only after confirming no live process owns it. - retry after an interrupted holder has exited; the kernel releases its lock
even though the `.lock` metadata file remains for inspection.
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
## Restore conflict without `--force` ## Restore conflict without `--force`
@@ -156,6 +798,8 @@ Symptom:
Likely cause: Likely cause:
- local durable files differ from remote restore sources. - local durable files differ from remote restore sources.
- a restore target or one of its ancestors is a symlink or another non-regular
filesystem entry.
Diagnostics: Diagnostics:
@@ -166,7 +810,13 @@ narratio session restore 2026-04-04 --dry-run
Safe fix: Safe fix:
- review conflicts; - review conflicts;
- rerun with `--force` only when remote state should overwrite local. - rerun with `--force` only when remote state should overwrite an eligible
regular local file; repair a symlink or other non-regular conflict first.
If the filesystem changed after the plan was printed, rerun restore after
repairing the destination; installation performs the same safety check again.
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
## Restore current-state discovery failure ## Restore current-state discovery failure
@@ -191,6 +841,8 @@ Safe fix:
- resolve storage/auth issue; - resolve storage/auth issue;
- republish from healthy local state if current pointer is missing. - republish from healthy local state if current pointer is missing.
Relevant reference: [Operations: Publish Workflow](./operations.md#publish-workflow).
## Publish output failure ## Publish output failure
Symptom: Symptom:
@@ -217,6 +869,8 @@ Safe fix:
- correct publish source/destination rules; - correct publish source/destination rules;
- retry after storage failure is resolved. - retry after storage failure is resolved.
Relevant reference: [Publish configuration](./config.md#publish-configuration-summary).
## Render markdown source missing ## Render markdown source missing
Symptom: Symptom:
@@ -232,7 +886,6 @@ Diagnostics:
```bash ```bash
narratio session status 2026-04-04 narratio session status 2026-04-04
narratio run-stage render 2026-04-04 --force
``` ```
Safe fix: Safe fix:
@@ -244,6 +897,8 @@ narratio run-stage render 2026-04-04 --force
narratio run-stage analyze 2026-04-04 --force narratio run-stage analyze 2026-04-04 --force
``` ```
Relevant reference: [Operations: Stage Execution](./operations.md#stage-execution-and-continuation-behavior).
## Secrets or storage credential failure ## Secrets or storage credential failure
Symptom: Symptom:
@@ -252,7 +907,8 @@ Symptom:
Likely causes: Likely causes:
- invalid `pipeline.secrets.env_dir`; - missing, unreadable, or invalid explicit `--secrets-dir`;
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
- missing credential environment variables; - missing credential environment variables;
- invalid S3 endpoint/bucket settings. - invalid S3 endpoint/bucket settings.
@@ -260,15 +916,17 @@ Diagnostics:
```bash ```bash
ls -la /path/to/secrets_dir ls -la /path/to/secrets_dir
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM' env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
``` ```
Safe fix: Safe fix:
- correct secret-file path and permissions; - correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
- provide required env vars; - provide required env vars;
- keep secret values out of YAML. - keep secret values out of YAML.
Relevant reference: [Secrets](./config.md#secrets-handling).
## S3 audio prepare failure ## S3 audio prepare failure
Symptom: Symptom:
@@ -284,7 +942,7 @@ Likely causes:
Diagnostics: Diagnostics:
```bash ```bash
narratio run-stage prepare 2026-04-04 --force narratio session validate 2026-04-04
``` ```
Safe fix: Safe fix:
@@ -292,6 +950,8 @@ Safe fix:
- verify prefix contents and storage access; - verify prefix contents and storage access;
- keep session audio mode consistent. - keep session audio mode consistent.
Relevant reference: [Operations](./operations.md).
## References ## References
- [docs/cli.md](./cli.md) - [docs/cli.md](./cli.md)

79
examples/README.md Normal file
View File

@@ -0,0 +1,79 @@
# Maintained Examples
These files are safe, copyable starting points for Narratio configuration and
input structure. Replace placeholder identifiers, storage names, integration
URLs, and paths for the target environment. Field meanings and defaults belong
in the [configuration reference](../docs/config.md).
## Pipeline Configuration
- [Minimal pipeline](pipeline.minimal.yml): campaign discovery plus the required
WhisperX URL.
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
external tools, and configured Scriptorium artifacts.
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
one family per `artifact-families/<key>.yml`, and the fixed
`profiles/testing.yml` overlay. Production values live in the base. The
bundle demonstrates an input-only artifact dependency, two character families
sharing the campaign party, and a testing-only disabled artifact. Inspect it
offline with:
```sh
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
```
These inspection commands load configuration and resolve explicit credential
references, but do not load a session or create runtime state. This bundle is
intentionally local-storage and credential-reference free so the walkthrough
needs no tools or cloud services.
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
pipeline section with explanatory comments.
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
Scriptorium artifact consuming only three declared Notarius lanes.
The existing `internal/config` example test loads and validates each pipeline
with the sample campaign and a compatible local- or S3-audio session.
## Campaign And Session Configuration
- [Sample campaign](campaigns/sample-campaign/campaign.yml), its
[session template](campaigns/sample-campaign/session.template.yml), and its
adjacent stable inputs provide a complete campaign directory shape.
- [Local-audio session](session.local-audio.yml) and
[S3-audio session](session.s3-audio.yml) are concrete session files.
- [Session template](session.template.yml) and the campaign-local equivalent
demonstrate the narrow placeholder syntax consumed by `session init`; they
are templates, not runtime session files.
## Input Fixtures
- [Speakers](speakers.yml), [autocorrect](autocorrect.yml), and
[glossary](glossary.yml) show the standalone input shapes.
- The sample campaign references its local
[speakers](campaigns/sample-campaign/speakers.yml),
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
follows the Notarius v0.7 contract. Narratio derives the players projection
from this party source.
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
reserves the expected filename and directory shape. Replace it with a real
FLAC file before running transcription.
The examples contain environment-variable names but no credential values. Set
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
examples that reference them. They use fictional campaign content and reserved
example domains.
For a deployed bundle, keep credential files out of source control in a
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
environment-variable name, or select another directory with `--secrets-dir`.
Static S3 credentials are optional: omit both S3 credential references to use
the AWS SDK discovery chain.

View File

@@ -4,3 +4,5 @@ inputs:
speakers_file: ./speakers.yml speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml glossary_file: ./glossary.yml
party_file: ./party.yml
spell_catalog_file: ./spell_catalog.json

View File

@@ -0,0 +1,26 @@
schema_version: narratio.party.v1
characters:
arannis:
player:
name: Rowan Hale
character:
name: Arannis
alias:
- Ari
- The Grey Owl
classes:
- name: wizard
level: 8
brenna:
player:
name: Rowan Hale
character:
name: Brenna
alias:
- Shield of Dawn
classes:
- name: paladin
level: 6
- name: warlock
level: 2

View File

@@ -1,5 +1,5 @@
match: match:
- speaker: "Eric Rakestraw" - speaker: "Example Speaker"
match: match:
- "Eric_Rakestraw" - "Example_Speaker"
- "Eric" - "Example"

View File

@@ -0,0 +1,18 @@
{
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
"catalogs": [
{
"id": "narratio.sample-campaign",
"ruleset": "dnd-5e-2014",
"source": {
"title": "Narratio sample campaign spell names"
},
"spells": [
{
"name": "Aegis of Emberfall",
"aliases": ["Emberfall Aegis"]
}
]
}
]
}

View File

@@ -0,0 +1,59 @@
# Purpose-specific extraction example: a Scriptorium session brief consumes
# only the three Notarius lanes it needs.
campaigns:
root: /usr/local/share/narratio/campaigns
default_campaign_id: sample-campaign
whisperx:
transcribe_url: "https://transcription.example.com/transcribe"
notarius:
enabled: true
binary: notarius
config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session
timeout: 3h
references:
glossary: narratio.input.glossary
party: narratio.input.party
players: narratio.input.players
spell_catalog: narratio.input.spell_catalog
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
location_registry:
lane_id: location-registry
media_type: application/json
schema_id: notarius.dnd.location_registry
schema_version: v1
module_key: dnd/location-registry
scene_descriptions:
lane_id: scene-descriptions
media_type: application/json
schema_id: notarius.dnd.scene_descriptions
schema_version: v1
module_key: dnd/scene-descriptions
scriptorium:
binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml
artifacts:
session_brief:
enabled: true
prompt_id: dnd.session_brief
output_path: artifacts/session_brief.md
inputs:
npcs:
source: narratio.extraction.npc_registry
required: true
locations:
source: narratio.extraction.location_registry
required: true
scenes:
source: narratio.extraction.scene_descriptions
required: true

View File

@@ -7,12 +7,11 @@ workspace:
# Optional: remove run-scoped workdir after successful publish commit. # Optional: remove run-scoped workdir after successful publish commit.
cleanup_after_publish: false cleanup_after_publish: false
# Optional: local secret file loader (directory of ENV_VAR_NAME files). # Credential files, when used, belong in a sibling secrets/ directory or in the
# secrets: # directory selected with --secrets-dir; they are not pipeline configuration.
# env_dir: ./secrets
storage: storage:
# Optional storage backend selector; use "s3" for publish + S3 audio workflows. # Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
backend: s3 backend: s3
s3: s3:
# Required when using S3 audio or S3 publish uploads. # Required when using S3 audio or S3 publish uploads.
@@ -24,8 +23,8 @@ storage:
endpoint: "" endpoint: ""
force_path_style: false force_path_style: false
# Optional; defaults shown explicitly. # Optional; defaults shown explicitly.
access_key_id_env: OBJECT_STORAGE_KEY_ID access_key_id: OBJECT_STORAGE_KEY_ID
secret_access_key_env: OBJECT_STORAGE_KEY secret_access_key: OBJECT_STORAGE_KEY
campaigns: campaigns:
# Optional; defaults to /usr/local/share/narratio/campaigns. # Optional; defaults to /usr/local/share/narratio/campaigns.
@@ -60,6 +59,11 @@ publish:
- source: narratio.artifact.player_handout - source: narratio.artifact.player_handout
dest: artifacts/player_handout.md dest: artifacts/player_handout.md
required: false required: false
# Extraction lanes publish only when named explicitly; the bundle and index
# are never implicit publish sources.
- source: narratio.extraction.npc_registry
dest: artifacts/extraction/npc-registry.json
required: true
whisperx: whisperx:
# Required. # Required.
@@ -89,7 +93,7 @@ audita:
# Optional overrides; defaults shown explicitly where applicable. # Optional overrides; defaults shown explicitly where applicable.
binary: audita binary: audita
timeout: 3h timeout: 3h
llm_api_key_env: AUDITA_LLM_API_KEY llm_api_key: AUDITA_LLM_API_KEY
modules: [glossary, homophones, spoken_word, grammar] modules: [glossary, homophones, spoken_word, grammar]
base_url: "" base_url: ""
model: "" model: ""
@@ -110,20 +114,111 @@ normalize:
report: true report: true
trim: trim:
# Keep disabled unless bounds prompt integration is configured. # Optional; defaults shown explicitly.
enabled: false enabled: true
output_path: transcripts/final.trimmed.json output_path: transcripts/final.trimmed.json
bounds: bounds:
prompt_id: dnd.session_bounds prompt_id: dnd.session_bounds
profile_id: local-fast profile_id: ""
transcript_input_name: transcript transcript_input_name: transcript
output_path: reports/session_bounds.json output_path: artifacts/session_bounds.json
timeout: 10m timeout: 10m
render_debug: false render_debug: false
render_output_path: reports/session_bounds.render.json
seriatim: seriatim:
report: false report: false
notarius:
# Optional structured extraction after render and before analyze.
enabled: true
binary: notarius
# Optional environment-variable reference. Match this name to api_key_env in
# the selected Notarius/PromptKit profile; never put the credential here.
llm_api_key: NOTARIUS_LLM_API_KEY
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
# is independent of Narratio's selected profile and may use a different name.
# See docs/config.md and docs/integrations/notarius.md for the contracts.
# config_profile: downstream-economy
config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session
timeout: 3h
working_directory: /usr/local/etc/notarius
# Reuse compatible accepted Notarius checkpoints during executed extraction.
# The Notarius config must enable recording and provide persistent cache roots.
resume: true
# auto uses or creates a source-addressed chunk plan. Forced Narratio
# extraction overrides this with refresh and disables checkpoint loading.
chunk_cache: auto
# External campaign references use prepared Narratio source IDs. Omit an
# optional binding when the selected Notarius pipeline does not need it.
references:
glossary: narratio.input.glossary
party: narratio.input.party
players: narratio.input.players
spell_catalog: narratio.input.spell_catalog
# Each key creates source narratio.extraction.<key>. These constraints match
# the current Notarius D&D lane contracts; update them with Notarius.
outputs:
item_registry:
lane_id: item-registry
media_type: application/json
schema_id: notarius.dnd.item_registry
schema_version: v1
module_key: dnd/item-registry
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
location_registry:
lane_id: location-registry
media_type: application/json
schema_id: notarius.dnd.location_registry
schema_version: v1
module_key: dnd/location-registry
scene_descriptions:
lane_id: scene-descriptions
media_type: application/json
schema_id: notarius.dnd.scene_descriptions
schema_version: v1
module_key: dnd/scene-descriptions
item_occurrences:
lane_id: item-occurrences
media_type: application/json
schema_id: notarius.dnd.item_occurrences
schema_version: v1
module_key: dnd/item-occurrences
spells:
lane_id: spells
media_type: application/json
schema_id: notarius.dnd.spells
schema_version: v1
module_key: dnd/spells
combat_turns:
lane_id: combat-turns
media_type: application/json
schema_id: notarius.dnd.combat_turns
schema_version: v1
module_key: dnd/combat-turns
npc_occurrences:
lane_id: npc-occurrences
media_type: application/json
schema_id: notarius.dnd.npc_occurrences
schema_version: v1
module_key: dnd/npc-occurrences
location_occurrences:
lane_id: location-occurrences
media_type: application/json
schema_id: notarius.dnd.location_occurrences
schema_version: v1
module_key: dnd/location-occurrences
enemy_events:
lane_id: enemy-events
media_type: application/json
schema_id: notarius.dnd.enemy_events
schema_version: v1
module_key: dnd/enemy-events
scriptorium: scriptorium:
binary: scriptorium binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml config_path: /usr/local/etc/scriptorium/config.yml
@@ -144,6 +239,15 @@ scriptorium:
previous_recap: previous_recap:
source: narratio.previous_session.artifact.session_recap source: narratio.previous_session.artifact.session_recap
required: false required: false
players:
source: narratio.input.players
required: true
party:
source: narratio.input.party
required: true
glossary:
source: narratio.input.glossary
required: false
vars: vars:
session_id: true session_id: true
session_date: true session_date: true
@@ -175,7 +279,5 @@ scriptorium:
output_kind: player_handout output_kind: player_handout
notification: notification:
# Optional notification settings. # No delivery provider is currently implemented.
backend: "" mode: noop
recipient: ""
timeout: 30s

View File

@@ -8,8 +8,8 @@ storage:
bucket: my-dnd-archive bucket: my-dnd-archive
root_prefix: dnd root_prefix: dnd
region: us-east-1 region: us-east-1
access_key_id_env: OBJECT_STORAGE_KEY_ID access_key_id: OBJECT_STORAGE_KEY_ID
secret_access_key_env: OBJECT_STORAGE_KEY secret_access_key: OBJECT_STORAGE_KEY
campaigns: campaigns:
root: /usr/local/share/narratio/campaigns root: /usr/local/share/narratio/campaigns
@@ -57,7 +57,7 @@ seriatim:
audita: audita:
binary: audita binary: audita
timeout: 3h timeout: 3h
llm_api_key_env: AUDITA_LLM_API_KEY llm_api_key: AUDITA_LLM_API_KEY
modules: [glossary, homophones, spoken_word, grammar] modules: [glossary, homophones, spoken_word, grammar]
output_schema: audita-v1 output_schema: audita-v1
work_dir_retention: auto work_dir_retention: auto
@@ -71,9 +71,6 @@ normalize:
output_schema: seriatim-intermediate output_schema: seriatim-intermediate
report: true report: true
trim:
enabled: false
scriptorium: scriptorium:
binary: scriptorium binary: scriptorium
config_path: /usr/local/etc/scriptorium/config.yml config_path: /usr/local/etc/scriptorium/config.yml
@@ -93,6 +90,15 @@ scriptorium:
previous_recap: previous_recap:
source: narratio.previous_session.artifact.session_recap source: narratio.previous_session.artifact.session_recap
required: false required: false
players:
source: narratio.input.players
required: true
party:
source: narratio.input.party
required: true
glossary:
source: narratio.input.glossary
required: false
vars: vars:
session_id: true session_id: true
session_date: true session_date: true
@@ -119,4 +125,4 @@ scriptorium:
output_kind: player_handout output_kind: player_handout
notification: notification:
timeout: 30s mode: noop

View File

@@ -0,0 +1,20 @@
scriptorium:
artifact_families:
character_items:
enabled: true
for_each: party.characters
prompt_id: dnd.character_items
profile_id: production-placeholder
output_path_pattern: artifacts/characters/{character_id}/items.md
inputs:
character_meta:
source: narratio.member_artifact.character_meta
required: true
member_vars:
character_id: character_id
character_name: character.name
aliases: character.alias_summary
publish:
enabled: true
required: false
dest_pattern: artifacts/characters/{character_id}/items.md

View File

@@ -0,0 +1,13 @@
scriptorium:
artifact_families:
character_meta:
enabled: true
for_each: party.characters
prompt_id: dnd.character_meta
profile_id: production-placeholder
output_path_pattern: artifacts/characters/{character_id}/meta.md
member_vars:
character_id: character_id
character_name: character.name
player_name: player.name
class_summary: character.class_summary

View File

@@ -0,0 +1,11 @@
scriptorium:
artifacts:
player_handout:
enabled: true
prompt_id: dnd.player_handout
profile_id: production-placeholder
output_path: artifacts/player-handout.md
inputs:
recap:
source: narratio.artifact.session_recap
required: true

View File

@@ -0,0 +1,11 @@
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
profile_id: production-placeholder
output_path: artifacts/session-recap.md
inputs:
transcript:
source: narratio.transcript.final_trimmed
required: true

View File

@@ -0,0 +1,12 @@
workspace:
root: ./workspace
campaigns:
root: ../../campaigns
default_campaign_id: sample-campaign
cache:
root: ./cache
spool:
root: ./spool

View File

@@ -0,0 +1,7 @@
publish:
enabled: true
upload_run: false
outputs:
- source: narratio.transcript.final_markdown
dest: transcripts/final.md
required: true

View File

@@ -0,0 +1,2 @@
storage:
backend: local

View File

@@ -0,0 +1,18 @@
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
seriatim:
binary: seriatim
output_schema: seriatim-intermediate
audita:
binary: audita
modules: [glossary, grammar]
output_schema: audita-v1
model: narratio-production-model-placeholder
validation_model: narratio-production-validator-placeholder
normalize:
output_path: transcripts/final.json
output_schema: seriatim-intermediate

View File

@@ -0,0 +1,14 @@
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
# these explicit path and glob imports; Narratio never scans directories itself.
configkit:
imports:
- path: conf.d/platform.yml
- path: conf.d/storage.yml
- path: conf.d/transcript.yml
- path: conf.d/publish.yml
- glob: artifacts/*.yml
- glob: artifact-families/*.yml
scriptorium:
binary: scriptorium
config_path: ./scriptorium/config.yml

View File

@@ -0,0 +1,18 @@
audita:
model: narratio-testing-model-placeholder
validation_model: narratio-testing-validator-placeholder
scriptorium:
artifacts:
session_recap:
profile_id: testing-placeholder
player_handout:
profile_id: testing-placeholder
testing_notes:
enabled: false
output_path: artifacts/testing-notes.md
artifact_families:
character_meta:
profile_id: testing-placeholder
character_items:
profile_id: testing-placeholder

View File

@@ -1,5 +1,5 @@
match: match:
- speaker: "Eric Rakestraw" - speaker: "Example Speaker"
match: match:
- "Eric_Rakestraw" - "Example_Speaker"
- "Eric" - "Example"

7
go.mod
View File

@@ -1,12 +1,16 @@
module gitea.maximumdirect.net/eric/narratio module gitea.maximumdirect.net/eric/narratio
go 1.25.0 go 1.26.0
require ( require (
gitea.maximumdirect.net/eric/ConfigKit v0.3.1
github.com/aws/aws-sdk-go-v2/config v1.32.17 github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/smithy-go v1.25.1 github.com/aws/smithy-go v1.25.1
github.com/google/go-cmp v0.7.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
@@ -25,4 +29,5 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
) )

10
go.sum
View File

@@ -1,3 +1,5 @@
gitea.maximumdirect.net/eric/ConfigKit v0.3.1 h1:etfTI/dxO3NnS18La0V6IeUh/Iv+3PjVfEyaN8hTuOM=
gitea.maximumdirect.net/eric/ConfigKit v0.3.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
@@ -34,6 +36,14 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -4,10 +4,9 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"path/filepath" "path/filepath"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// NoopRunner is a deterministic no-op audita adapter. // NoopRunner is a deterministic no-op audita adapter.
@@ -84,17 +83,17 @@ func materializePlaceholders(req PolishRequest) error {
"merged_transcript_path": req.MergedTranscriptPath, "merged_transcript_path": req.MergedTranscriptPath,
"output_path": req.OutputProcessedPath, "output_path": req.OutputProcessedPath,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }
@@ -117,14 +116,14 @@ func writeJSONIfRequested(path string, payload any) error {
if path == "" { if path == "" {
return nil return nil
} }
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err) return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
} }
data, err := json.Marshal(payload) data, err := json.Marshal(payload)
if err != nil { if err != nil {
return fmt.Errorf("marshal placeholder json for %q: %w", path, err) return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
} }
if err := subprocess.WriteFileAtomic(path, data, 0o644); err != nil { if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write placeholder json %q: %w", path, err) return fmt.Errorf("write placeholder json %q: %w", path, err)
} }
return nil return nil

View File

@@ -0,0 +1,19 @@
package audita
import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gopkg.in/yaml.v3"
)
func writeGeneratedYAML(path string, payload any) error {
data, err := yaml.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
}
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated YAML %q: %w", path, err)
}
return nil
}

View File

@@ -6,8 +6,6 @@ import (
"time" "time"
) )
// TODO: implement a real Audita subprocess/service adapter.
// Runner is the adapter boundary for audita polish invocations. // Runner is the adapter boundary for audita polish invocations.
type Runner interface { type Runner interface {
Run(ctx context.Context, req PolishRequest) (PolishResult, error) Run(ctx context.Context, req PolishRequest) (PolishResult, error)
@@ -15,25 +13,15 @@ type Runner interface {
// PolishRequest describes an audita invocation. // PolishRequest describes an audita invocation.
type PolishRequest struct { type PolishRequest struct {
GeneratedConfigPath string GeneratedConfigPath string
MergedTranscriptPath string MergedTranscriptPath string
OutputProcessedPath string OutputProcessedPath string
GlossaryPath string GlossaryPath string
ReportPath string ReportPath string
WorkDir string WorkDir string
Modules []string Modules []string
BaseURL string StdoutLogPath string
Model string StderrLogPath string
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
StdoutLogPath string
StderrLogPath string
} }
// PolishResult describes a polish output. // PolishResult describes a polish output.

View File

@@ -5,19 +5,26 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/url" "net/url"
"os"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
// MaxReportOutputBytes bounds Audita's optional report JSON result.
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution. // SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
type SubprocessRunnerConfig struct { type SubprocessRunnerConfig struct {
Binary string Binary string
Timeout time.Duration Timeout time.Duration
LLMAPIKeyEnv string LLMAPIKey string
LLMAPIKeyReference string
Modules []string Modules []string
BaseURL string BaseURL string
Model string Model string
@@ -36,7 +43,8 @@ type SubprocessRunnerConfig struct {
type SubprocessRunner struct { type SubprocessRunner struct {
binary string binary string
timeout time.Duration timeout time.Duration
llmAPIKeyEnv string llmAPIKey string
llmAPIKeyReference string
modules []string modules []string
baseURL string baseURL string
model string model string
@@ -55,7 +63,8 @@ type SubprocessRunner struct {
func NewSubprocessRunnerFromConfigValues( func NewSubprocessRunnerFromConfigValues(
binary string, binary string,
timeout string, timeout string,
llmAPIKeyEnv string, llmAPIKey string,
llmAPIKeyReference string,
modules []string, modules []string,
baseURL string, baseURL string,
model string, model string,
@@ -79,7 +88,8 @@ func NewSubprocessRunnerFromConfigValues(
return NewSubprocessRunner(SubprocessRunnerConfig{ return NewSubprocessRunner(SubprocessRunnerConfig{
Binary: binary, Binary: binary,
Timeout: parsedTimeout, Timeout: parsedTimeout,
LLMAPIKeyEnv: llmAPIKeyEnv, LLMAPIKey: llmAPIKey,
LLMAPIKeyReference: llmAPIKeyReference,
Modules: modules, Modules: modules,
BaseURL: baseURL, BaseURL: baseURL,
Model: model, Model: model,
@@ -103,6 +113,9 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
if cfg.Timeout <= 0 { if cfg.Timeout <= 0 {
return nil, fmt.Errorf("audita timeout must be > 0") return nil, fmt.Errorf("audita timeout must be > 0")
} }
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
return nil, fmt.Errorf("audita credential value and reference must be provided together")
}
for i, module := range cfg.Modules { for i, module := range cfg.Modules {
if strings.TrimSpace(module) == "" { if strings.TrimSpace(module) == "" {
return nil, fmt.Errorf("audita module at index %d is empty", i) return nil, fmt.Errorf("audita module at index %d is empty", i)
@@ -145,7 +158,8 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
return &SubprocessRunner{ return &SubprocessRunner{
binary: strings.TrimSpace(cfg.Binary), binary: strings.TrimSpace(cfg.Binary),
timeout: cfg.Timeout, timeout: cfg.Timeout,
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv), llmAPIKey: cfg.LLMAPIKey,
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
modules: modules, modules: modules,
baseURL: strings.TrimSpace(cfg.BaseURL), baseURL: strings.TrimSpace(cfg.BaseURL),
model: strings.TrimSpace(cfg.Model), model: strings.TrimSpace(cfg.Model),
@@ -188,16 +202,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
} }
args := r.buildArgs(req, reqModules) args := r.buildArgs(req, reqModules)
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv) credentialPresent := r.llmAPIKeyReference != ""
credentialPresent := false
env := map[string]string{} env := map[string]string{}
if credentialEnvVar != "" { if credentialPresent {
credential, ok := os.LookupEnv(credentialEnvVar) env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
if !ok || strings.TrimSpace(credential) == "" {
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
}
env["AUDITA_LLM_API_KEY"] = credential
credentialPresent = true
} }
if req.GeneratedConfigPath != "" { if req.GeneratedConfigPath != "" {
@@ -207,12 +215,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
} }
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{ runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: r.binary, Executable: r.binary,
Args: args, Args: args,
Timeout: r.timeout, Timeout: r.timeout,
EnvOverrides: env, EnvOverrides: env,
StdoutLogPath: req.StdoutLogPath, DiagnosticOwner: "audita",
StderrLogPath: req.StderrLogPath, StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
if err != nil { if err != nil {
wrappedMessage := fmt.Sprintf( wrappedMessage := fmt.Sprintf(
@@ -261,7 +270,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
"total_llm_concurrency": r.totalLLMConcurrency, "total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency, "proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency, "validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv, "credential_env_var": r.llmAPIKeyReference,
"credential_present": credentialPresent, "credential_present": credentialPresent,
}, },
}, nil }, nil
@@ -291,7 +300,7 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
"total_llm_concurrency": r.totalLLMConcurrency, "total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency, "proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency, "validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv, "credential_env_var": r.llmAPIKeyReference,
"credential_present": credentialPresent, "credential_present": credentialPresent,
}, },
} }
@@ -367,16 +376,16 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
"output_path": req.OutputProcessedPath, "output_path": req.OutputProcessedPath,
"report_path": req.ReportPath, "report_path": req.ReportPath,
"work_dir": req.WorkDir, "work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv, "credential_env_var": r.llmAPIKeyReference,
"credential_present": credentialPresent, "credential_present": credentialPresent,
} }
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644) return writeGeneratedYAML(req.GeneratedConfigPath, payload)
} }
func validateProcessedOutput(path string) error { func validateProcessedOutput(path string) error {
data, err := os.ReadFile(path) data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
if err != nil { if err != nil {
return fmt.Errorf("read file: %w", err) return err
} }
var payload map[string]any var payload map[string]any
@@ -405,9 +414,9 @@ func addSubprocessStreamHint(message string, runErr error) string {
} }
func validateJSONFile(path string) error { func validateJSONFile(path string) error {
data, err := os.ReadFile(path) data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
if err != nil { if err != nil {
return fmt.Errorf("read file: %w", err) return err
} }
var v any var v any
if err := json.Unmarshal(data, &v); err != nil { if err := json.Unmarshal(data, &v); err != nil {
@@ -415,3 +424,11 @@ func validateJSONFile(path string) error {
} }
return nil return nil
} }
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
data, err := fileops.ReadFileBounded(path, limit)
if err != nil {
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
}
return data, nil
}

View File

@@ -6,16 +6,12 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"strings" "strings"
"testing" "testing"
"time" "time"
) )
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) { func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success") t.Setenv("AUDITA_HELPER_MODE", "success")
@@ -31,7 +27,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{ runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
Binary: wrapper, Binary: wrapper,
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary", "homophones", "glossary"}, Modules: []string{"glossary", "homophones", "glossary"},
BaseURL: "https://openrouter.ai/api/v1", BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it", Model: "openrouter/google/gemma-4-31b-it",
@@ -48,6 +45,7 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewSubprocessRunner() error = %v", err) t.Fatalf("NewSubprocessRunner() error = %v", err)
} }
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
dir := t.TempDir() dir := t.TempDir()
req := PolishRequest{ req := PolishRequest{
@@ -132,47 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
} }
} }
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) { func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
if runtime.GOOS == "windows" { _, err := NewSubprocessRunner(SubprocessRunnerConfig{
t.Skip("helper wrapper script uses /bin/sh") Binary: writeAuditaHelperWrapper(t),
} Timeout: mustParseAuditaDuration(t, "2s"),
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ LLMAPIKeyReference: "MISSING_AUDITA_KEY",
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
}) })
req := auditaReqForTest(t, false) if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
t.Fatalf("error = %q, want missing credential guidance", err.Error())
} }
} }
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) { func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success") t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json") recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath) t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "", LLMAPIKey: "",
Modules: []string{"glossary"}, Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1", BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it", Model: "openrouter/google/gemma-4-31b-it",
Report: false, Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
@@ -189,10 +171,7 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
} }
} }
func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) { func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success") t.Setenv("AUDITA_HELPER_MODE", "success")
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent") t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
@@ -200,13 +179,13 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath) t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
Modules: []string{"glossary"}, Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1", BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it", Model: "openrouter/google/gemma-4-31b-it",
Report: false, Report: false,
LLMAPIKeyEnv: "", LLMAPIKey: "",
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil { if _, err := runner.Run(context.Background(), req); err != nil {
@@ -214,27 +193,24 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
} }
rec := readAuditaHelperRecord(t, recordPath) rec := readAuditaHelperRecord(t, recordPath)
if rec.Env["AUDITA_INHERITED_MARKER"] != "inherited-from-parent" { if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want inherited-from-parent", rec.Env["AUDITA_INHERITED_MARKER"]) t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
} }
} }
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) { func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success") t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json") recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath) t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "", LLMAPIKey: "",
BaseURL: "https://openrouter.ai/api/v1", BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it", Model: "openrouter/google/gemma-4-31b-it",
Report: false, Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil { if _, err := runner.Run(context.Background(), req); err != nil {
@@ -249,19 +225,16 @@ func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
} }
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) { func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success") t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json") recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath) t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "", LLMAPIKey: "",
Report: false, Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil { if _, err := runner.Run(context.Background(), req); err != nil {
@@ -279,22 +252,20 @@ func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T
} }
func TestSubprocessRunnerSubprocessFailure(t *testing.T) { func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "fail") t.Setenv("AUDITA_HELPER_MODE", "fail")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: true, Model: "openrouter/google/gemma-4-31b-it",
Report: true,
}) })
req := auditaReqForTest(t, true) req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -310,25 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) { if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error()) t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
} }
if strings.Contains(err.Error(), "super-secret") {
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
}
} }
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) { func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd") t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: true, Model: "openrouter/google/gemma-4-31b-it",
Report: true,
}) })
req := auditaReqForTest(t, true) req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -341,22 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
} }
func TestSubprocessRunnerMissingOutputFails(t *testing.T) { func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "missing_output") t.Setenv("AUDITA_HELPER_MODE", "missing_output")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: false, Model: "openrouter/google/gemma-4-31b-it",
Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -369,22 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
} }
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) { func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "invalid_output") t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: false, Model: "openrouter/google/gemma-4-31b-it",
Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -397,22 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) { func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "segments_missing") t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: false, Model: "openrouter/google/gemma-4-31b-it",
Report: false,
}) })
req := auditaReqForTest(t, false) req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -425,22 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
} }
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) { func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1") t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "invalid_report") t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret") t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{ runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t), Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"), Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE", LLMAPIKey: "super-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
BaseURL: "https://openrouter.ai/api/v1", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "https://openrouter.ai/api/v1",
Report: true, Model: "openrouter/google/gemma-4-31b-it",
Report: true,
}) })
req := auditaReqForTest(t, true) req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req) _, err := runner.Run(context.Background(), req)
@@ -453,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerConstructorValidation(t *testing.T) { func TestSubprocessRunnerConstructorValidation(t *testing.T) {
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true) _, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil { if err == nil {
t.Fatal("expected binary validation error") t.Fatal("expected binary validation error")
} }
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true) _, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil { if err == nil {
t.Fatal("expected timeout parse error") t.Fatal("expected timeout parse error")
} }
_, err = NewSubprocessRunner(SubprocessRunnerConfig{ _, err = NewSubprocessRunner(SubprocessRunnerConfig{
Binary: "audita", Binary: "audita",
Timeout: mustParseAuditaDuration(t, "1s"), Timeout: mustParseAuditaDuration(t, "1s"),
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY", LLMAPIKey: "synthetic-secret",
Modules: []string{"glossary"}, LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
BaseURL: "://", Modules: []string{"glossary"},
Model: "openrouter/google/gemma-4-31b-it", BaseURL: "://",
Model: "openrouter/google/gemma-4-31b-it",
}) })
if err == nil { if err == nil {
t.Fatal("expected base url validation error") t.Fatal("expected base url validation error")

View File

@@ -0,0 +1,28 @@
package notarius
import "context"
// FakeRunner is a configurable in-memory runner for stage tests.
type FakeRunner struct {
Requests []RunRequest
Result RunResult
Err error
}
// Run records the request and returns the configured result or error.
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
if err := ctx.Err(); err != nil {
return RunResult{}, err
}
copyRequest := req
if req.ConfigProfile != nil {
profile := *req.ConfigProfile
copyRequest.ConfigProfile = &profile
}
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
f.Requests = append(f.Requests, copyRequest)
if f.Err != nil {
return RunResult{}, f.Err
}
return f.Result, nil
}

View File

@@ -0,0 +1,167 @@
// Package notarius declares the adapter contract for Notarius CLI invocations.
package notarius
import (
"context"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
const ReceiptSchemaVersion = "notarius.run-result.v2"
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
type Runner interface {
Run(ctx context.Context, req RunRequest) (RunResult, error)
}
// ReferenceBinding maps one normalized Notarius selector to an absolute
// external reference path.
type ReferenceBinding struct {
Selector string
Path string
}
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
type RunRequest struct {
Binary string
// LLMAPIKey is an optional already-resolved credential reference.
LLMAPIKey *configkit.Secret
ConfigProfile *string
ConfigPath string
PipelineID string
InputPath string
OutputRoot string
WorkingDirectory string
ReceiptPath string
LogPath string
Timeout time.Duration
Resume bool
ChunkCache string
References []ReferenceBinding
}
// Receipt is the transport-neutral successful run receipt.
type Receipt struct {
SchemaVersion string
RunID string
PipelineID string
OutputDirectory string
IndexFile string
NormalizedOutputCount int
RejectedOutputCount int
WarningGroupCount int
WarningOccurrenceCount int
DiagnosticGroupCount int
DiagnosticOccurrenceCount int
DiagnosticsTruncated bool
ValidationStatus string
ValidationSummaries []ValidationSummary
DebugDirectory string
}
// ValidationSummary retains the bounded outcome of one Notarius producer result.
type ValidationSummary struct {
Stage string
StepID string
LaneID string
ModuleKey string
ChunkID string
Status string
RejectingValidators []string
ReasonCodes []string
IncompleteValidators []string
ProducerAttemptCount int
TerminalAction string
}
// LaneDescriptor identifies one normalized lane payload discovered through the index.
type LaneDescriptor struct {
LaneID string
File string
Path string
MediaType string
ModuleKey string
SchemaID string
SchemaName string
SchemaVersion string
}
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
type PipelineDescriptor struct {
ArtifactKind string
File string
Path string
MediaType string
SchemaID string
SchemaName string
SchemaVersion string
}
// Index describes the validated bundle-management and artifact paths.
type Index struct {
Path string
ManifestFile string
ManifestPath string
RejectedFile string
RejectedPath string
WarningsFile string
WarningsPath string
DiagnosticsFile string
DiagnosticsPath string
Lanes []LaneDescriptor
ChunkMap *PipelineDescriptor
EvidenceContext *PipelineDescriptor
}
// RejectionSummary retains structured rejection identity without free-form messages.
type RejectionSummary struct {
Stage string
StepID string
LaneID string
ModuleKey string
ChunkID string
ValidatorName string
ReasonCode string
}
// WarningSummary retains structured warning identity without free-form messages.
type WarningSummary struct {
Disposition string
Category string
ReasonCode string
Origin DiagnosticOrigin
OccurrenceCount int
}
// DiagnosticOrigin identifies the framework-owned pipeline location of a finding.
type DiagnosticOrigin struct {
Stage string
StepID string
LaneID string
ModuleKey string
ValidatorKey string
}
// DiagnosticSummary retains bounded advisory or observation group metadata.
type DiagnosticSummary struct {
Disposition string
Category string
ReasonCode string
Origin DiagnosticOrigin
OccurrenceCount int
}
// RunResult describes a successfully decoded and validated Notarius bundle.
type RunResult struct {
Receipt Receipt
Index Index
BundleRoot string
ReceiptPath string
LogPath string
ExitCode int
Duration time.Duration
Rejections []RejectionSummary
Warnings []WarningSummary
Diagnostics []DiagnosticSummary
}

View File

@@ -0,0 +1,821 @@
package notarius
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/notariusref"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
const (
maxReceiptBytes = 1 << 20
maxIndexBytes = 4 << 20
maxSummaryBytes = 4 << 20
canonicalIndexFile = "index.json"
canonicalManifestFile = "manifest.json"
canonicalRejectedFile = "rejected.json"
canonicalWarningsFile = "warnings.json"
canonicalDiagnosticsFile = "diagnostics.json"
warningsSchemaVersion = "notarius.warnings.v2"
diagnosticsSchemaVersion = "notarius.diagnostics.v1"
maxWarningGroups = 128
maxDiagnosticGroups = 256
maxFindingSamples = 3
)
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
// SubprocessRunner invokes Notarius through its public CLI.
type SubprocessRunner struct {
run subprocessRun
}
// NewSubprocessRunner constructs a production Notarius subprocess runner.
func NewSubprocessRunner() *SubprocessRunner {
return &SubprocessRunner{run: subprocess.Run}
}
// Run executes a complete Notarius pipeline and discovers its published bundle.
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
if r == nil || r.run == nil {
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
}
references, err := validateRunRequest(req)
if err != nil {
return RunResult{}, err
}
var envOverrides map[string]string
var sensitiveEnvNames []string
if req.LLMAPIKey != nil {
if !req.LLMAPIKey.IsResolved() {
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
}
name := req.LLMAPIKey.EnvName()
value := req.LLMAPIKey.Value()
if value == "" {
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
}
envOverrides = map[string]string{name: value}
sensitiveEnvNames = []string{name}
}
args := []string{
"run", req.PipelineID,
"--config", req.ConfigPath,
"--input", req.InputPath,
"--output-dir", req.OutputRoot,
}
if req.Resume {
args = append(args, "--resume")
}
if req.ChunkCache != "" {
args = append(args, "--chunk_cache", req.ChunkCache)
}
for _, reference := range references {
args = append(args, "--reference", reference.Selector+"="+reference.Path)
}
if req.ConfigProfile != nil {
args = append(args, "--config-profile", *req.ConfigProfile)
}
args = append(args, "--json")
processResult, err := r.run(ctx, subprocess.RunRequest{
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDirectory,
Timeout: req.Timeout,
DiagnosticOwner: "notarius",
StdoutLogPath: req.ReceiptPath,
StderrLogPath: req.LogPath,
EnvOverrides: envOverrides,
SensitiveEnvNames: sensitiveEnvNames,
})
baseResult := RunResult{
ReceiptPath: req.ReceiptPath,
LogPath: req.LogPath,
ExitCode: processResult.ExitCode,
Duration: processResult.Duration,
}
if err != nil {
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
}
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
if err != nil {
return baseResult, err
}
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
if err != nil {
return baseResult, err
}
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
if err != nil {
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
}
index, err := loadIndex(bundleRoot, indexPath)
if err != nil {
return baseResult, err
}
rejections, err := loadRejections(index.RejectedPath)
if err != nil {
return baseResult, err
}
warnings, err := loadWarnings(index.WarningsPath)
if err != nil {
return baseResult, err
}
diagnostics, diagnosticOccurrences, diagnosticsTruncated, err := loadDiagnostics(index.DiagnosticsPath)
if err != nil {
return baseResult, err
}
if receipt.NormalizedOutputCount != len(index.Lanes) || receipt.RejectedOutputCount != len(rejections) ||
receipt.WarningGroupCount != len(warnings) || receipt.DiagnosticGroupCount != len(diagnostics) {
return baseResult, fmt.Errorf("notarius receipt counts do not match published bundle")
}
warningOccurrences, err := sumWarningOccurrences(warnings)
if err != nil {
return baseResult, err
}
if receipt.WarningOccurrenceCount != warningOccurrences ||
receipt.DiagnosticOccurrenceCount != diagnosticOccurrences ||
receipt.DiagnosticsTruncated != diagnosticsTruncated {
return baseResult, fmt.Errorf("notarius receipt occurrence counts do not match published bundle")
}
baseResult.Receipt = receipt
baseResult.Index = index
baseResult.BundleRoot = bundleRoot
baseResult.Rejections = rejections
baseResult.Warnings = warnings
baseResult.Diagnostics = diagnostics
return baseResult, nil
}
func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
if strings.TrimSpace(req.Binary) == "" {
return nil, fmt.Errorf("notarius binary is required")
}
if strings.TrimSpace(req.PipelineID) == "" {
return nil, fmt.Errorf("notarius pipeline id is required")
}
if req.Timeout <= 0 {
return nil, fmt.Errorf("notarius timeout must be positive")
}
if !validChunkCacheMode(req.ChunkCache) {
return nil, fmt.Errorf("notarius chunk cache mode must be empty, auto, bypass, or refresh")
}
for label, path := range map[string]string{
"config": req.ConfigPath,
"input": req.InputPath,
"output root": req.OutputRoot,
"working directory": req.WorkingDirectory,
"receipt": req.ReceiptPath,
"log": req.LogPath,
} {
if strings.TrimSpace(path) == "" {
return nil, fmt.Errorf("notarius %s path is required", label)
}
if !filepath.IsAbs(path) {
return nil, fmt.Errorf("notarius %s path must be absolute", label)
}
}
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
return nil, fmt.Errorf("notarius receipt and log paths must be different")
}
references := make([]ReferenceBinding, 0, len(req.References))
selectors := make(map[string]struct{}, len(req.References))
for index, binding := range req.References {
selector, err := notariusref.NormalizeSelector(binding.Selector)
if err != nil {
return nil, fmt.Errorf("notarius reference %d selector: %w", index, err)
}
if _, duplicate := selectors[selector]; duplicate {
return nil, fmt.Errorf("notarius reference selector %q is duplicated", selector)
}
selectors[selector] = struct{}{}
if strings.TrimSpace(binding.Path) == "" {
return nil, fmt.Errorf("notarius reference %q path is required", selector)
}
if !filepath.IsAbs(binding.Path) {
return nil, fmt.Errorf("notarius reference %q path must be absolute", selector)
}
references = append(references, ReferenceBinding{Selector: selector, Path: binding.Path})
}
if err := requireRegularFile(req.ConfigPath); err != nil {
return nil, fmt.Errorf("validate notarius config path: %w", err)
}
if err := requireRegularFile(req.InputPath); err != nil {
return nil, fmt.Errorf("validate notarius input path: %w", err)
}
if err := requireDirectory(req.OutputRoot); err != nil {
return nil, fmt.Errorf("validate notarius output root: %w", err)
}
if err := requireDirectory(req.WorkingDirectory); err != nil {
return nil, fmt.Errorf("validate notarius working directory: %w", err)
}
if err := validateLogDestination(req.ReceiptPath); err != nil {
return nil, fmt.Errorf("validate notarius receipt path: %w", err)
}
if err := validateLogDestination(req.LogPath); err != nil {
return nil, fmt.Errorf("validate notarius log path: %w", err)
}
return references, nil
}
func validChunkCacheMode(mode string) bool {
switch mode {
case "", "auto", "bypass", "refresh":
return true
default:
return false
}
}
type receiptDocument struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file"`
NormalizedOutputCount *int `json:"normalized_output_count"`
RejectedOutputCount *int `json:"rejected_output_count"`
WarningGroupCount *int `json:"warning_group_count"`
WarningOccurrenceCount *int `json:"warning_occurrence_count"`
DiagnosticGroupCount *int `json:"diagnostic_group_count"`
DiagnosticOccurrenceCount *int `json:"diagnostic_occurrence_count"`
DiagnosticsTruncated *bool `json:"diagnostics_truncated"`
ValidationStatus string `json:"validation_status"`
ValidationSummaries []validationSummaryDocument `json:"validation_summaries"`
DebugDirectory string `json:"debug_directory"`
}
type validationSummaryDocument struct {
Stage string `json:"stage"`
StepID string `json:"step_id"`
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
ChunkID string `json:"chunk_id"`
Status string `json:"status"`
RejectingValidators []string `json:"rejecting_validators"`
ReasonCodes []string `json:"reason_codes"`
IncompleteValidators []string `json:"incomplete_validators"`
ProducerAttemptCount *int `json:"producer_attempt_count"`
TerminalAction string `json:"terminal_action"`
}
func validValidationStatus(value string) bool {
switch value {
case "approved", "rejected", "incomplete":
return true
default:
return false
}
}
func validateValidationSummaries(documents []validationSummaryDocument) ([]ValidationSummary, error) {
summaries := make([]ValidationSummary, 0, len(documents))
for _, document := range documents {
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
}
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount < 0 || !validTerminalAction(document.TerminalAction) {
return nil, fmt.Errorf("notarius validation summary is missing required fields")
}
summaries = append(summaries, ValidationSummary{
Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID,
ModuleKey: document.ModuleKey, ChunkID: document.ChunkID, Status: document.Status,
RejectingValidators: append([]string(nil), document.RejectingValidators...),
ReasonCodes: append([]string(nil), document.ReasonCodes...),
IncompleteValidators: append([]string(nil), document.IncompleteValidators...),
ProducerAttemptCount: *document.ProducerAttemptCount, TerminalAction: document.TerminalAction,
})
}
return summaries, nil
}
func validTerminalAction(value string) bool {
switch value {
case "accepted", "reject_output", "warn_continue", "fail_run":
return true
default:
return false
}
}
func loadReceipt(path, pipelineID string) (Receipt, error) {
var document receiptDocument
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
}
if document.SchemaVersion != ReceiptSchemaVersion {
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
}
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
document.NormalizedOutputCount == nil ||
document.RejectedOutputCount == nil || document.WarningGroupCount == nil ||
document.WarningOccurrenceCount == nil || document.DiagnosticGroupCount == nil ||
document.DiagnosticOccurrenceCount == nil || document.DiagnosticsTruncated == nil {
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
}
if document.IndexFile != canonicalIndexFile {
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
}
if document.PipelineID != pipelineID {
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
}
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 ||
*document.WarningGroupCount < 0 || *document.WarningOccurrenceCount < 0 ||
*document.DiagnosticGroupCount < 0 || *document.DiagnosticOccurrenceCount < 0 {
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
}
if !validValidationStatus(document.ValidationStatus) {
return Receipt{}, fmt.Errorf("notarius receipt validation_status %q is invalid", document.ValidationStatus)
}
validationSummaries, err := validateValidationSummaries(document.ValidationSummaries)
if err != nil {
return Receipt{}, err
}
if !filepath.IsAbs(document.OutputDirectory) {
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
}
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
}
return Receipt{
SchemaVersion: document.SchemaVersion,
RunID: document.RunID,
PipelineID: document.PipelineID,
OutputDirectory: filepath.Clean(document.OutputDirectory),
IndexFile: document.IndexFile,
NormalizedOutputCount: *document.NormalizedOutputCount,
RejectedOutputCount: *document.RejectedOutputCount,
WarningGroupCount: *document.WarningGroupCount,
WarningOccurrenceCount: *document.WarningOccurrenceCount,
DiagnosticGroupCount: *document.DiagnosticGroupCount,
DiagnosticOccurrenceCount: *document.DiagnosticOccurrenceCount,
DiagnosticsTruncated: *document.DiagnosticsTruncated,
ValidationStatus: document.ValidationStatus,
ValidationSummaries: validationSummaries,
DebugDirectory: document.DebugDirectory,
}, nil
}
type indexDocument struct {
ManifestFile string `json:"manifest_file"`
OutputFiles *[]laneDocument `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
DiagnosticsFile string `json:"diagnostics_file"`
ChunkMap *pipelineDocument `json:"chunk_map"`
EvidenceContext *pipelineDocument `json:"evidence_context"`
}
type laneDocument struct {
LaneID string `json:"lane_id"`
File string `json:"file"`
MediaType string `json:"media_type"`
ModuleKey string `json:"module_key"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
}
type pipelineDocument struct {
ArtifactKind string `json:"artifact_kind"`
File string `json:"file"`
MediaType string `json:"media_type"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
}
func loadIndex(bundleRoot, indexPath string) (Index, error) {
var document indexDocument
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
return Index{}, fmt.Errorf("decode notarius index: %w", err)
}
for _, field := range []struct {
name string
got string
want string
}{
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
{name: "diagnostics_file", got: document.DiagnosticsFile, want: canonicalDiagnosticsFile},
} {
if field.got != field.want {
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
}
}
if document.OutputFiles == nil {
return Index{}, fmt.Errorf("notarius index is missing required output_files")
}
index := Index{
Path: indexPath,
ManifestFile: document.ManifestFile,
RejectedFile: document.RejectedFile,
WarningsFile: document.WarningsFile,
DiagnosticsFile: document.DiagnosticsFile,
}
var err error
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
}
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
}
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
}
if index.DiagnosticsPath, err = resolveRegularFile(bundleRoot, index.DiagnosticsFile); err != nil {
return Index{}, fmt.Errorf("resolve notarius diagnostics file: %w", err)
}
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
for _, lane := range *document.OutputFiles {
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
}
if _, exists := seenLanes[lane.LaneID]; exists {
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
}
seenLanes[lane.LaneID] = struct{}{}
path, err := resolveRegularFile(bundleRoot, lane.File)
if err != nil {
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
}
index.Lanes = append(index.Lanes, LaneDescriptor{
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
SchemaVersion: lane.SchemaVersion,
})
}
if document.ChunkMap != nil {
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
if err != nil {
return Index{}, err
}
}
if document.EvidenceContext != nil {
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
if err != nil {
return Index{}, err
}
}
return index, nil
}
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
}
path, err := resolveRegularFile(bundleRoot, document.File)
if err != nil {
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
}
return &PipelineDescriptor{
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
MediaType: document.MediaType, SchemaID: document.SchemaID,
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
}, nil
}
type rejectionDocument struct {
Rejected *[]struct {
Stage string `json:"stage"`
StepID string `json:"step_id"`
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
ChunkID string `json:"chunk_id"`
ValidatorName string `json:"validator_name"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
} `json:"rejected"`
}
func loadRejections(path string) ([]RejectionSummary, error) {
var document rejectionDocument
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
return nil, fmt.Errorf("decode notarius rejections: %w", err)
}
if document.Rejected == nil {
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
}
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
for _, item := range *document.Rejected {
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
return nil, fmt.Errorf("notarius rejection entries require stage and message")
}
summaries = append(summaries, RejectionSummary{
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
})
}
return summaries, nil
}
type findingGroupDocument struct {
Disposition string `json:"disposition"`
Category string `json:"category"`
ReasonCode string `json:"reason_code"`
Origin diagnosticOriginDocument `json:"origin"`
OccurrenceCount *int `json:"occurrence_count"`
Samples *[]struct {
Scope string `json:"scope"`
Message string `json:"message"`
ChunkID string `json:"chunk_id"`
ChunkIndex *int `json:"chunk_index"`
} `json:"samples"`
OmittedSampleCount *int `json:"omitted_sample_count"`
}
type diagnosticOriginDocument struct {
Stage string `json:"stage"`
StepID string `json:"step_id"`
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
ValidatorKey string `json:"validator_key"`
}
type warningDocument struct {
SchemaVersion string `json:"schema_version"`
GroupCount *int `json:"group_count"`
OccurrenceCount *int `json:"occurrence_count"`
Groups *[]findingGroupDocument `json:"groups"`
}
func loadWarnings(path string) ([]WarningSummary, error) {
var document warningDocument
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
return nil, fmt.Errorf("decode notarius warnings: %w", err)
}
if document.SchemaVersion != warningsSchemaVersion || document.GroupCount == nil ||
document.OccurrenceCount == nil || document.Groups == nil {
return nil, fmt.Errorf("notarius warning document is missing or incompatible required fields")
}
if *document.GroupCount < 0 || *document.GroupCount > maxWarningGroups || *document.OccurrenceCount < 0 ||
*document.GroupCount != len(*document.Groups) {
return nil, fmt.Errorf("notarius warning document counts are inconsistent")
}
summaries := make([]WarningSummary, 0, len(*document.Groups))
occurrences := 0
for _, group := range *document.Groups {
if err := validateFindingGroup(group); err != nil {
return nil, fmt.Errorf("notarius warning group: %w", err)
}
if group.Disposition != "warning" {
return nil, fmt.Errorf("notarius warning group disposition %q is invalid", group.Disposition)
}
if *group.OccurrenceCount > int(^uint(0)>>1)-occurrences {
return nil, fmt.Errorf("notarius warning occurrence count overflows")
}
occurrences += *group.OccurrenceCount
summaries = append(summaries, WarningSummary{
Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode,
Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount,
})
}
if occurrences != *document.OccurrenceCount {
return nil, fmt.Errorf("notarius warning document occurrence count is inconsistent")
}
return summaries, nil
}
type diagnosticDocument struct {
SchemaVersion string `json:"schema_version"`
GroupCount *int `json:"group_count"`
OccurrenceCount *int `json:"occurrence_count"`
Truncated *bool `json:"truncated"`
UnrepresentedOccurrenceCount *int `json:"unrepresented_occurrence_count"`
Groups *[]findingGroupDocument `json:"groups"`
}
func loadDiagnostics(path string) ([]DiagnosticSummary, int, bool, error) {
var document diagnosticDocument
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
return nil, 0, false, fmt.Errorf("decode notarius diagnostics: %w", err)
}
if document.SchemaVersion != diagnosticsSchemaVersion || document.GroupCount == nil ||
document.OccurrenceCount == nil || document.Truncated == nil ||
document.UnrepresentedOccurrenceCount == nil || document.Groups == nil {
return nil, 0, false, fmt.Errorf("notarius diagnostics document is missing or incompatible required fields")
}
if *document.GroupCount < 0 || *document.GroupCount > maxDiagnosticGroups || *document.OccurrenceCount < 0 ||
*document.UnrepresentedOccurrenceCount < 0 || *document.GroupCount != len(*document.Groups) {
return nil, 0, false, fmt.Errorf("notarius diagnostics document counts are inconsistent")
}
if !*document.Truncated && *document.UnrepresentedOccurrenceCount != 0 {
return nil, 0, false, fmt.Errorf("notarius diagnostics document has unrepresented occurrences without truncation")
}
summaries := make([]DiagnosticSummary, 0, len(*document.Groups))
representedOccurrences := 0
for _, group := range *document.Groups {
if err := validateFindingGroup(group); err != nil {
return nil, 0, false, fmt.Errorf("notarius diagnostic group: %w", err)
}
if group.Disposition != "advisory" && group.Disposition != "observation" {
return nil, 0, false, fmt.Errorf("notarius diagnostic group disposition %q is invalid", group.Disposition)
}
if *group.OccurrenceCount > int(^uint(0)>>1)-representedOccurrences {
return nil, 0, false, fmt.Errorf("notarius diagnostic occurrence count overflows")
}
representedOccurrences += *group.OccurrenceCount
summaries = append(summaries, DiagnosticSummary{
Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode,
Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount,
})
}
if *document.UnrepresentedOccurrenceCount > int(^uint(0)>>1)-representedOccurrences ||
representedOccurrences+*document.UnrepresentedOccurrenceCount != *document.OccurrenceCount {
return nil, 0, false, fmt.Errorf("notarius diagnostics document occurrence count is inconsistent")
}
return summaries, *document.OccurrenceCount, *document.Truncated, nil
}
func validateFindingGroup(group findingGroupDocument) error {
if strings.TrimSpace(group.Disposition) == "" || strings.TrimSpace(group.Category) == "" ||
strings.TrimSpace(group.ReasonCode) == "" || !validDiagnosticOriginStage(group.Origin.Stage) ||
!validDiagnosticCategory(group.Disposition, group.Category) ||
group.OccurrenceCount == nil || *group.OccurrenceCount <= 0 || group.Samples == nil ||
group.OmittedSampleCount == nil || *group.OmittedSampleCount < 0 {
return fmt.Errorf("missing required fields")
}
if len(*group.Samples) == 0 || len(*group.Samples) > maxFindingSamples ||
*group.OmittedSampleCount != *group.OccurrenceCount-len(*group.Samples) {
return fmt.Errorf("sample counts are inconsistent")
}
for _, sample := range *group.Samples {
if strings.TrimSpace(sample.Scope) == "" || strings.TrimSpace(sample.Message) == "" ||
(sample.ChunkIndex != nil && *sample.ChunkIndex < 0) {
return fmt.Errorf("samples require scope and message")
}
}
return nil
}
func validDiagnosticCategory(disposition, category string) bool {
switch disposition {
case "warning":
return category == "configuration" || category == "degradation" ||
category == "validation_incomplete" || category == "fallback"
case "advisory":
return category == "data_quality"
case "observation":
return category == "normalization"
default:
return false
}
}
func validDiagnosticOriginStage(stage string) bool {
switch stage {
case "references", "chunk", "extract", "merge", "normalize":
return true
default:
return false
}
}
func diagnosticOrigin(document diagnosticOriginDocument) DiagnosticOrigin {
return DiagnosticOrigin{
Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID,
ModuleKey: document.ModuleKey, ValidatorKey: document.ValidatorKey,
}
}
func sumWarningOccurrences(values []WarningSummary) (int, error) {
total := 0
for _, value := range values {
if value.OccurrenceCount > int(^uint(0)>>1)-total {
return 0, fmt.Errorf("notarius warning occurrence count overflows")
}
total += value.OccurrenceCount
}
return total, nil
}
func decodeBoundedJSON(path string, limit int64, destination any) error {
data, err := fileops.ReadFileBounded(path, limit)
if err != nil {
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
}
if err := json.Unmarshal(data, destination); err != nil {
return err
}
return nil
}
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
root := filepath.Clean(outputRoot)
bundle := filepath.Clean(bundleRoot)
relative, err := filepath.Rel(root, bundle)
if err != nil {
return "", fmt.Errorf("compare notarius output paths: %w", err)
}
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
}
if err := requireDirectoryTree(root, relative); err != nil {
return "", fmt.Errorf("validate notarius output directory: %w", err)
}
return bundle, nil
}
func resolveRegularFile(root, logicalPath string) (string, error) {
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
if err != nil {
return "", err
}
relative, err := filepath.Rel(root, resolved)
if err != nil {
return "", err
}
if err := requireRegularFileTree(root, relative); err != nil {
return "", err
}
return resolved, nil
}
func requireDirectoryTree(root, relative string) error {
if err := requireDirectory(root); err != nil {
return err
}
current := root
for _, component := range strings.Split(relative, string(filepath.Separator)) {
current = filepath.Join(current, component)
if err := requireDirectory(current); err != nil {
return err
}
}
return nil
}
func requireRegularFileTree(root, relative string) error {
components := strings.Split(relative, string(filepath.Separator))
if len(components) == 0 {
return fmt.Errorf("regular file path is required")
}
if err := requireDirectory(root); err != nil {
return err
}
current := root
for _, component := range components[:len(components)-1] {
current = filepath.Join(current, component)
if err := requireDirectory(current); err != nil {
return err
}
}
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
}
func requireDirectory(path string) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return fmt.Errorf("path %q must be a directory without symlinks", path)
}
return nil
}
func requireRegularFile(path string) error {
info, err := os.Lstat(path)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf("path %q must be a regular file without symlinks", path)
}
return nil
}
func validateLogDestination(path string) error {
if err := requireDirectory(filepath.Dir(path)); err != nil {
return err
}
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
}
return nil
}

View File

@@ -0,0 +1,979 @@
package notarius
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
)
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
req := validRunRequest(t)
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, true)
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
}}
result, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
wantArgs := []string{
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot, "--json",
}
if !reflect.DeepEqual(captured.Args, wantArgs) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
}
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
t.Fatalf("subprocess request = %#v", captured)
}
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
}
if captured.EnvOverrides != nil {
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
}
for _, arg := range captured.Args {
if arg == "--session-id" {
t.Fatal("subprocess args unexpectedly contain --session-id")
}
}
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
t.Fatalf("receipt = %#v", result.Receipt)
}
if len(result.Receipt.ValidationSummaries) != 1 || result.Receipt.ValidationSummaries[0].LaneID != "npc-registry" ||
result.Receipt.ValidationSummaries[0].Status != "complete" {
t.Fatalf("validation summaries = %#v", result.Receipt.ValidationSummaries)
}
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
t.Fatalf("lanes = %#v", result.Index.Lanes)
}
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
}
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
}
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
t.Fatalf("rejections = %#v", result.Rejections)
}
if len(result.Warnings) != 1 || result.Warnings[0].Category != "degradation" || result.Warnings[0].ReasonCode != "normalized_name" {
t.Fatalf("warnings = %#v", result.Warnings)
}
if len(result.Diagnostics) != 1 || result.Diagnostics[0].Category != "data_quality" || result.Diagnostics[0].ReasonCode != "low_confidence" {
t.Fatalf("diagnostics = %#v", result.Diagnostics)
}
}
func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
tests := []struct {
name string
resume bool
chunkCache string
want []string
}{
{name: "resume", resume: true, want: []string{"--resume"}},
{name: "auto", chunkCache: "auto", want: []string{"--chunk_cache", "auto"}},
{name: "bypass", chunkCache: "bypass", want: []string{"--chunk_cache", "bypass"}},
{name: "refresh", chunkCache: "refresh", want: []string{"--chunk_cache", "refresh"}},
{name: "resume and mode", resume: true, chunkCache: "auto", want: []string{"--resume", "--chunk_cache", "auto"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
req.Resume = test.resume
req.ChunkCache = test.chunkCache
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
want := []string{
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot,
}
want = append(want, test.want...)
want = append(want, "--json")
if !reflect.DeepEqual(captured.Args, want) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, want)
}
})
}
}
func TestSubprocessRunnerMapsConfigProfileToArgumentPair(t *testing.T) {
tests := []struct {
name string
supplied bool
value string
}{
{name: "omitted"},
{name: "empty", supplied: true},
{name: "default", supplied: true, value: "default"},
{name: "named", supplied: true, value: "inexpensive"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
if test.supplied {
req.ConfigProfile = &test.value
}
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
flagIndex := -1
for index, arg := range captured.Args {
if arg == "--config-profile" {
flagIndex = index
break
}
}
if !test.supplied {
if flagIndex >= 0 {
t.Fatalf("subprocess args unexpectedly contain config profile: %#v", captured.Args)
}
return
}
if flagIndex < 0 || flagIndex+2 >= len(captured.Args) || captured.Args[flagIndex+1] != test.value || captured.Args[flagIndex+2] != "--json" {
t.Fatalf("config profile argument boundary = %#v, want flag, exact value, then --json", captured.Args)
}
})
}
}
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
for _, mode := range []string{"AUTO", "invalid", " auto "} {
t.Run(mode, func(t *testing.T) {
req := validRunRequest(t)
req.ChunkCache = mode
started := false
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
started = true
return sharedsubprocess.RunResult{}, nil
}}
_, err := runner.Run(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "chunk cache mode") {
t.Fatalf("Run() error = %v, want chunk cache validation failure", err)
}
if started {
t.Fatal("subprocess started with invalid chunk cache mode")
}
})
}
}
func TestSubprocessRunnerForwardsLoadedCredentialWithoutRereadingEnvironment(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_LLM_KEY"
const loadedValue = "loaded-notarius-credential"
credential := loadTestCredential(t, envName, loadedValue)
t.Setenv(envName, "changed-parent-credential")
req := validRunRequest(t)
req.LLMAPIKey = credential
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(captured.EnvOverrides) != 1 || captured.EnvOverrides[envName] != loadedValue {
t.Fatal("subprocess did not receive exactly the loaded credential override")
}
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{envName}) {
t.Fatalf("sensitive environment names = %#v, want configured reference name", captured.SensitiveEnvNames)
}
wantArgs := []string{
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot, "--json",
}
if !reflect.DeepEqual(captured.Args, wantArgs) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
}
}
func TestSubprocessRunnerRejectsInvalidProgrammaticCredentialBeforeLaunch(t *testing.T) {
for _, test := range []struct {
name string
reference configkit.Secret
}{
{name: "unresolved", reference: configkit.SecretEnv("NARRATIO_NOTARIUS_UNRESOLVED_KEY")},
{name: "empty", reference: configkit.SecretEnv("")},
} {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
req.LLMAPIKey = &test.reference
started := false
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
started = true
return sharedsubprocess.RunResult{}, nil
}}
_, err := runner.Run(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "resolved and nonempty") {
t.Fatalf("Run() error = %v, want credential validation failure", err)
}
if started {
t.Fatal("subprocess started with an invalid credential")
}
})
}
}
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
req := validRunRequest(t)
referenceRoot := t.TempDir()
req.References = []ReferenceBinding{
{Selector: " party ", Path: filepath.Join(referenceRoot, "party context=primary.json")},
{Selector: " npc-registry . extract . glossary ", Path: filepath.Join(referenceRoot, "glossary.json")},
}
originalReferences := append([]ReferenceBinding(nil), req.References...)
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
wantArgs := []string{
"run", req.PipelineID,
"--config", req.ConfigPath,
"--input", req.InputPath,
"--output-dir", req.OutputRoot,
"--reference", "party=" + req.References[0].Path,
"--reference", "npc-registry.extract.glossary=" + req.References[1].Path,
"--json",
}
if !reflect.DeepEqual(captured.Args, wantArgs) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
}
if !reflect.DeepEqual(req.References, originalReferences) {
t.Fatalf("Run() mutated caller references = %#v, want %#v", req.References, originalReferences)
}
}
func TestSubprocessRunnerRejectsInvalidReferencesBeforeLaunch(t *testing.T) {
tests := []struct {
name string
references func(string) []ReferenceBinding
wantErr string
}{
{
name: "invalid selector",
references: func(root string) []ReferenceBinding {
return []ReferenceBinding{{Selector: "lane.prepare.party", Path: filepath.Join(root, "party.json")}}
},
wantErr: "selector",
},
{
name: "duplicate normalized selector",
references: func(root string) []ReferenceBinding {
return []ReferenceBinding{
{Selector: "lane.party", Path: filepath.Join(root, "party.json")},
{Selector: " lane . party ", Path: filepath.Join(root, "party-2.json")},
}
},
wantErr: "duplicated",
},
{
name: "empty path",
references: func(string) []ReferenceBinding {
return []ReferenceBinding{{Selector: "party", Path: " "}}
},
wantErr: "path is required",
},
{
name: "relative path",
references: func(string) []ReferenceBinding {
return []ReferenceBinding{{Selector: "party", Path: "references/party.json"}}
},
wantErr: "path must be absolute",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := validRunRequest(t)
req.References = tt.references(t.TempDir())
started := false
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
started = true
return sharedsubprocess.RunResult{}, nil
}}
_, err := runner.Run(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Run() error = %v, want containing %q", err, tt.wantErr)
}
if started {
t.Fatal("subprocess started after request validation failure")
}
})
}
}
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_KEY"
const canary = "notarius-captured-stream-canary"
const unrelated = "NARRATIO_NOTARIUS_UNRELATED_PARENT_KEY"
req := validRunRequest(t)
req.LLMAPIKey = loadTestCredential(t, envName, canary)
writeValidBundleAndReceipt(t, req, false)
receiptFixture := req.ReceiptPath + ".fixture"
data, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
var receipt map[string]any
if err := json.Unmarshal(data, &receipt); err != nil {
t.Fatalf("decode receipt fixture: %v", err)
}
receipt["credential_echo"] = canary
writeJSONFile(t, receiptFixture, receipt)
if err := os.Remove(req.ReceiptPath); err != nil {
t.Fatalf("Remove(receipt) error = %v", err)
}
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
if err := os.Mkdir(captureDir, 0o755); err != nil {
t.Fatalf("Mkdir(capture) error = %v", err)
}
script := writeShellScript(t, `#!/bin/sh
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
if [ "$`+envName+`" != '`+canary+`' ]; then
printf 'selected credential mismatch\n' >&2
exit 8
fi
if [ -n "$`+unrelated+`" ]; then
printf 'unrelated credential inherited\n' >&2
exit 9
fi
printf '%s\n' "$`+envName+`" >&2
cat "$NOTARIUS_RECEIPT_FIXTURE"
`)
req.Binary = script
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
t.Setenv(unrelated, "unrelated-parent-credential")
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
assertRedactedCredentialFile(t, req.LogPath, canary)
receiptBytes, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
assertRedactedCredentialBytes(t, receiptBytes, canary)
}
func TestSubprocessRunnerRedactsCredentialOnProcessFailure(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_KEY"
const canary = "notarius-failure-stream-canary"
req := validRunRequest(t)
req.LLMAPIKey = loadTestCredential(t, envName, canary)
req.Binary = writeShellScript(t, `#!/bin/sh
printf '%s\n' "$`+envName+`"
printf '%s\n' "$`+envName+`" >&2
exit 7
`)
_, err := NewSubprocessRunner().Run(context.Background(), req)
if err == nil {
t.Fatal("Run() error = nil, want process failure")
}
if strings.Contains(err.Error(), canary) || !strings.Contains(err.Error(), "<redacted>") {
t.Fatal("process failure did not return redacted diagnostics")
}
assertRedactedCredentialFile(t, req.ReceiptPath, canary)
assertRedactedCredentialFile(t, req.LogPath, canary)
}
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
tests := []struct {
name string
scriptBody string
timeout time.Duration
cancel bool
want string
}{
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
if test.name == "nonzero" {
profile := "missing-profile"
req.ConfigProfile = &profile
}
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
req.Timeout = test.timeout
ctx := context.Background()
if test.cancel {
cancelCtx, cancel := context.WithCancel(ctx)
ctx = cancelCtx
time.AfterFunc(20*time.Millisecond, cancel)
}
_, err := NewSubprocessRunner().Run(ctx, req)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
}
if strings.Contains(err.Error(), "decode notarius receipt") {
t.Fatalf("Run() parsed stdout after process failure: %v", err)
}
})
}
}
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
req := validRunRequest(t)
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
t.Fatalf("WriteFile(receipt) error = %v", err)
}
wantErr := errors.New("process failed")
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
}}
_, err := runner.Run(context.Background(), req)
if !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want wrapped process error", err)
}
if strings.Contains(err.Error(), "decode") {
t.Fatalf("Run() parsed receipt after failure: %v", err)
}
}
func TestLoadReceiptValidation(t *testing.T) {
root := t.TempDir()
valid := map[string]any{
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
"normalized_output_count": 1, "rejected_output_count": 0,
"warning_group_count": 0, "warning_occurrence_count": 0,
"diagnostic_group_count": 0, "diagnostic_occurrence_count": 0,
"diagnostics_truncated": false, "validation_status": "approved", "future_field": true,
}
tests := []struct {
name string
mutate func(map[string]any)
raw []byte
wantOK bool
wantError string
}{
{name: "unknown fields tolerated", wantOK: true},
{name: "malformed", raw: []byte("{")},
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v1" }},
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
{name: "negative count", mutate: func(v map[string]any) { v["warning_group_count"] = -1 }},
{name: "invalid validation status", mutate: func(v map[string]any) { v["validation_status"] = "valid" }},
{
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
wantError: `index_file "nested/index.json"`,
},
{
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
wantError: `index_file "./index.json"`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
values := cloneMap(valid)
if test.mutate != nil {
test.mutate(values)
}
if test.raw != nil {
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
} else {
writeJSONFile(t, path, values)
}
_, err := loadReceipt(path, "pipeline-1")
if test.wantOK && err != nil {
t.Fatalf("loadReceipt() error = %v", err)
}
if !test.wantOK && err == nil {
t.Fatal("loadReceipt() error = nil, want validation failure")
}
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
}
})
}
oversized := filepath.Join(root, "oversized.json")
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized) error = %v", err)
}
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadReceipt(oversized) error = %v", err)
}
}
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
root := t.TempDir()
outputRoot := filepath.Join(root, "output")
if err := os.Mkdir(outputRoot, 0o755); err != nil {
t.Fatalf("Mkdir(output root) error = %v", err)
}
validBundle := filepath.Join(outputRoot, "run-1")
if err := os.Mkdir(validBundle, 0o755); err != nil {
t.Fatalf("Mkdir(bundle) error = %v", err)
}
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
t.Fatalf("validateBundleRoot(valid) error = %v", err)
}
outside := filepath.Join(root, "output-other")
if err := os.Mkdir(outside, 0o755); err != nil {
t.Fatalf("Mkdir(outside) error = %v", err)
}
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
t.Run(name, func(t *testing.T) {
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
}
})
}
symlink := filepath.Join(outputRoot, "linked")
if err := os.Symlink(outside, symlink); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
t.Fatal("validateBundleRoot(symlink) error = nil")
}
}
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
tests := []struct {
name string
indexValue any
prepare func(*testing.T, string)
wantError string
}{
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
{name: "renamed manifest", indexValue: func() any {
value := validIndexValue([]any{})
value["manifest_file"] = "metadata.json"
return value
}(), wantError: `manifest_file "metadata.json"`},
{name: "cleanable manifest", indexValue: func() any {
value := validIndexValue([]any{})
value["manifest_file"] = "./manifest.json"
return value
}(), wantError: `manifest_file "./manifest.json"`},
{name: "renamed rejections", indexValue: func() any {
value := validIndexValue([]any{})
value["rejected_file"] = "rejections.json"
return value
}(), wantError: `rejected_file "rejections.json"`},
{name: "renamed warnings", indexValue: func() any {
value := validIndexValue([]any{})
value["warnings_file"] = "diagnostics/warnings.json"
return value
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
{name: "duplicate lane", indexValue: validIndexValue([]any{
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
})},
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
}},
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
t.Skipf("Symlink() unavailable: %v", err)
}
}},
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
t.Fatalf("Remove(manifest) error = %v", err)
}
}},
{name: "incomplete pipeline descriptor", indexValue: func() any {
value := validIndexValue([]any{})
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
return value
}()},
{name: "pipeline descriptor escape", indexValue: func() any {
value := validIndexValue([]any{})
value["evidence_context"] = map[string]any{
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
}
return value
}()},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
bundle := createBundleSkeleton(t)
indexPath := filepath.Join(bundle, "index.json")
if raw, ok := test.indexValue.(json.RawMessage); ok {
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
t.Fatalf("WriteFile(index) error = %v", err)
}
} else {
writeJSONFile(t, indexPath, test.indexValue)
}
if test.prepare != nil {
test.prepare(t, bundle)
}
if _, err := loadIndex(bundle, indexPath); err == nil {
t.Fatal("loadIndex() error = nil, want failure")
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
}
})
}
bundle := createBundleSkeleton(t)
oversizedIndex := filepath.Join(bundle, "index.json")
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized index) error = %v", err)
}
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadIndex(oversized) error = %v", err)
}
}
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
root := t.TempDir()
rejectedPath := filepath.Join(root, "rejected.json")
warningsPath := filepath.Join(root, "warnings.json")
diagnosticsPath := filepath.Join(root, "diagnostics.json")
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
}}, "future": true})
writeJSONFile(t, warningsPath, findingEnvelope(warningsSchemaVersion, []any{findingGroup("warning", "degradation", "bounded", "normalize", 2)}, 2, false, 0))
writeJSONFile(t, diagnosticsPath, findingEnvelope(diagnosticsSchemaVersion, []any{findingGroup("advisory", "data_quality", "low_confidence", "normalize", 3)}, 4, true, 1))
rejections, err := loadRejections(rejectedPath)
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
}
warnings, err := loadWarnings(warningsPath)
if err != nil || len(warnings) != 1 || warnings[0].Category != "degradation" || warnings[0].OccurrenceCount != 2 {
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
}
diagnostics, occurrences, truncated, err := loadDiagnostics(diagnosticsPath)
if err != nil || len(diagnostics) != 1 || occurrences != 4 || !truncated || diagnostics[0].Category != "data_quality" {
t.Fatalf("loadDiagnostics() = %#v, %d, %t, %v", diagnostics, occurrences, truncated, err)
}
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath, "diagnostics": diagnosticsPath} {
t.Run("malformed "+name, func(t *testing.T) {
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
var err error
if name == "rejections" {
_, err = loadRejections(path)
} else if name == "warnings" {
_, err = loadWarnings(path)
} else {
_, _, _, err = loadDiagnostics(path)
}
if err == nil {
t.Fatal("summary decoder error = nil")
}
})
}
oversized := filepath.Join(root, "oversized.json")
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
t.Fatalf("WriteFile(oversized) error = %v", err)
}
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadWarnings(oversized) error = %v", err)
}
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadRejections(oversized) error = %v", err)
}
if _, _, _, err := loadDiagnostics(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("loadDiagnostics(oversized) error = %v", err)
}
}
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
profile := "testing"
req := RunRequest{PipelineID: "pipeline", ConfigProfile: &profile, References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
want := RunResult{BundleRoot: "/bundle"}
fake := &FakeRunner{Result: want}
got, err := fake.Run(context.Background(), req)
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
}
req.References[0].Path = "/references/changed.json"
if fake.Requests[0].References[0].Path != "/references/party.json" {
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
}
profile = "changed"
if *fake.Requests[0].ConfigProfile != "testing" {
t.Fatalf("fake retained aliased config profile: %q", *fake.Requests[0].ConfigProfile)
}
wantErr := errors.New("configured failure")
fake.Err = wantErr
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
t.Fatalf("Run(configured error) = %v", err)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
before := len(fake.Requests)
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
}
}
func validRunRequest(t *testing.T) RunRequest {
t.Helper()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
inputPath := filepath.Join(root, "input.json")
outputRoot := filepath.Join(root, "outputs")
workingDirectory := filepath.Join(root, "work")
diagnostics := filepath.Join(root, "diagnostics")
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
if err := os.Mkdir(directory, 0o755); err != nil {
t.Fatalf("Mkdir(%q) error = %v", directory, err)
}
}
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
t.Fatalf("WriteFile(config) error = %v", err)
}
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
t.Fatalf("WriteFile(input) error = %v", err)
}
return RunRequest{
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
Timeout: time.Second,
}
}
func loadTestCredential(t *testing.T, envName, value string) *configkit.Secret {
t.Helper()
t.Setenv(envName, value)
type testConfig struct {
Credential *configkit.Secret `yaml:"credential"`
}
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+envName+"\n")), configkit.Options[testConfig]{})
if err != nil {
t.Fatalf("load test credential: %v", err)
}
return result.Config.Credential
}
func assertRedactedCredentialFile(t *testing.T, path, canary string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
assertRedactedCredentialBytes(t, data, canary)
}
func assertRedactedCredentialBytes(t *testing.T, data []byte, canary string) {
t.Helper()
text := string(data)
if strings.Contains(text, canary) || !strings.Contains(text, "<redacted>") {
t.Fatal("captured stream did not contain the required redaction marker")
}
}
func TestValidateValidationSummariesAcceptsReusedProducerWithoutAttempts(t *testing.T) {
zero := 0
summaries, err := validateValidationSummaries([]validationSummaryDocument{{
Stage: "chunk", Status: "complete", ProducerAttemptCount: &zero, TerminalAction: "accepted",
}})
if err != nil {
t.Fatalf("validateValidationSummaries() error = %v", err)
}
if len(summaries) != 1 || summaries[0].ProducerAttemptCount != 0 {
t.Fatalf("summaries = %#v, want one reused producer with zero attempts", summaries)
}
negative := -1
if _, err := validateValidationSummaries([]validationSummaryDocument{{
Stage: "chunk", Status: "complete", ProducerAttemptCount: &negative, TerminalAction: "accepted",
}}); err == nil {
t.Fatal("validateValidationSummaries() negative producer attempts error = nil")
}
}
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
t.Helper()
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
t.Fatalf("MkdirAll(bundle) error = %v", err)
}
for path, data := range map[string]string{
"manifest.json": `{}`,
"lanes/npc.json": `{}`,
"chunk-map.json": `{}`,
"evidence-context.json": `{}`,
} {
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
if includeUnknown {
rejection["future"] = true
}
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), findingEnvelope(warningsSchemaVersion, []any{findingGroup("warning", "degradation", "normalized_name", "normalize", 2)}, 2, false, 0))
writeJSONFile(t, filepath.Join(bundle, "diagnostics.json"), findingEnvelope(diagnosticsSchemaVersion, []any{findingGroup("advisory", "data_quality", "low_confidence", "normalize", 3)}, 4, true, 1))
index := validIndexValue([]any{map[string]any{
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
}})
index["chunk_map"] = map[string]any{
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
}
index["evidence_context"] = map[string]any{
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
}
index["future"] = true
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
receipt := map[string]any{
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
"rejected_output_count": 1, "warning_group_count": 1, "warning_occurrence_count": 2,
"diagnostic_group_count": 1, "diagnostic_occurrence_count": 4,
"diagnostics_truncated": true, "validation_status": "rejected",
"validation_summaries": []any{map[string]any{
"stage": "normalize", "lane_id": "npc-registry", "status": "complete",
"producer_attempt_count": 1, "terminal_action": "accepted",
}},
}
if includeUnknown {
receipt["future"] = true
}
writeJSONFile(t, req.ReceiptPath, receipt)
}
func createBundleSkeleton(t *testing.T) string {
t.Helper()
bundle := filepath.Join(t.TempDir(), "bundle")
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
t.Fatalf("MkdirAll(bundle) error = %v", err)
}
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "diagnostics.json", "lanes/npc.json", "chunk-map.json"} {
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", name, err)
}
}
return bundle
}
func validIndexValue(lanes []any) map[string]any {
return map[string]any{
"manifest_file": "manifest.json", "output_files": lanes,
"rejected_file": "rejected.json", "warnings_file": "warnings.json", "diagnostics_file": "diagnostics.json",
}
}
func findingGroup(disposition, category, reasonCode, origin string, occurrences int) map[string]any {
return map[string]any{
"disposition": disposition, "category": category, "reason_code": reasonCode,
"origin": map[string]any{"stage": origin, "lane_id": "npc-registry"}, "occurrence_count": occurrences,
"samples": []any{map[string]any{"scope": "lane:npc-registry", "message": "external detail"}},
"omitted_sample_count": occurrences - 1,
}
}
func findingEnvelope(schema string, groups []any, occurrences int, truncated bool, unrepresented int) map[string]any {
value := map[string]any{
"schema_version": schema, "group_count": len(groups), "occurrence_count": occurrences,
"groups": groups,
}
if schema == diagnosticsSchemaVersion {
value["truncated"] = truncated
value["unrepresented_occurrence_count"] = unrepresented
}
return value
}
func writeJSONFile(t *testing.T, path string, value any) {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}
func writeShellScript(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "notarius-helper")
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
t.Fatalf("WriteFile(script) error = %v", err)
}
return path
}
func assertTextFile(t *testing.T, path, want string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
if string(data) != want {
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
}
}
func cloneMap(source map[string]any) map[string]any {
result := make(map[string]any, len(source))
for key, value := range source {
result[key] = value
}
return result
}

View File

@@ -4,7 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// NoopRunner is a deterministic no-op scriptorium adapter. // NoopRunner is a deterministic no-op scriptorium adapter.
@@ -142,7 +142,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
func materializeRunPlaceholders(req RunArtifactRequest) error { func materializeRunPlaceholders(req RunArtifactRequest) error {
if req.OutputPath != "" { if req.OutputPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write run output %q: %w", req.OutputPath, err) return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
} }
} }
@@ -154,17 +154,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
"prompt_id": req.PromptID, "prompt_id": req.PromptID,
"output_path": req.OutputPath, "output_path": req.OutputPath,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }
@@ -173,7 +173,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
func materializeRenderPlaceholders(req RenderArtifactRequest) error { func materializeRenderPlaceholders(req RenderArtifactRequest) error {
if req.OutputPath != "" { if req.OutputPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write render output %q: %w", req.OutputPath, err) return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
} }
} }
@@ -185,17 +185,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
"prompt_id": req.PromptID, "prompt_id": req.PromptID,
"output_path": req.OutputPath, "output_path": req.OutputPath,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }

View File

@@ -0,0 +1,19 @@
package scriptorium
import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gopkg.in/yaml.v3"
)
func writeGeneratedYAML(path string, payload any) error {
data, err := yaml.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
}
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated YAML %q: %w", path, err)
}
return nil
}

View File

@@ -9,8 +9,12 @@ import (
"time" "time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// MaxOutputFileBytes bounds one Scriptorium artifact result.
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
// SubprocessRunner invokes Scriptorium through its public CLI. // SubprocessRunner invokes Scriptorium through its public CLI.
type SubprocessRunner struct{} type SubprocessRunner struct{}
@@ -52,13 +56,17 @@ func (r *SubprocessRunner) RunArtifact(ctx context.Context, req RunArtifactReque
} }
} }
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{ runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
Executable: req.Binary, Executable: req.Binary,
Args: args, Args: args,
WorkingDir: req.WorkingDir, WorkingDir: req.WorkingDir,
Timeout: req.Timeout, Timeout: req.Timeout,
StdoutLogPath: req.StdoutLogPath, EnvOverrides: envOverrides,
StderrLogPath: req.StderrLogPath, SensitiveEnvNames: sensitiveNames,
DiagnosticOwner: "scriptorium",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
result := ArtifactResult{ result := ArtifactResult{
@@ -134,13 +142,17 @@ func (r *SubprocessRunner) RenderArtifact(ctx context.Context, req RenderArtifac
} }
} }
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{ runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
Executable: req.Binary, Executable: req.Binary,
Args: args, Args: args,
WorkingDir: req.WorkingDir, WorkingDir: req.WorkingDir,
Timeout: req.Timeout, Timeout: req.Timeout,
StdoutLogPath: req.StdoutLogPath, EnvOverrides: envOverrides,
StderrLogPath: req.StderrLogPath, SensitiveEnvNames: sensitiveNames,
DiagnosticOwner: "scriptorium",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
result := ArtifactResult{ result := ArtifactResult{
@@ -223,6 +235,15 @@ func validateCommonRunRequest(
return true, nil return true, nil
} }
func credentialEnvironment(apiKeyEnv string) (map[string]string, []string) {
name := strings.TrimSpace(apiKeyEnv)
if name == "" {
return nil, nil
}
value, _ := os.LookupEnv(name)
return map[string]string{name: value}, []string{name}
}
func buildRunArgs(req RunArtifactRequest) []string { func buildRunArgs(req RunArtifactRequest) []string {
args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)} args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)}
if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" { if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" {
@@ -321,18 +342,15 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
"render_format": payload.RenderFormat, "render_format": payload.RenderFormat,
"render_prompt_logged": payload.RenderPromptStore, "render_prompt_logged": payload.RenderPromptStore,
} }
return subprocess.WriteYAMLAtomic(path, data, 0o644) return writeGeneratedYAML(path, data)
} }
func validateNonEmptyOutput(path string) error { func validateNonEmptyOutput(path string) error {
info, err := os.Stat(path) data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
if err != nil { if err != nil {
return fmt.Errorf("stat file: %w", err) return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
} }
if info.IsDir() { if len(data) == 0 {
return fmt.Errorf("path is a directory")
}
if info.Size() <= 0 {
return fmt.Errorf("file is empty") return fmt.Errorf("file is empty")
} }
return nil return nil

View File

@@ -6,16 +6,12 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"strings" "strings"
"testing" "testing"
"time" "time"
) )
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) { func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1") t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success") t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
} }
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) { func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1") t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1") t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
} }
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) { func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1") t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output") t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
} }
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) { func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1") t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output") t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
} }
func TestSubprocessRunnerRenderSuccess(t *testing.T) { func TestSubprocessRunnerRenderSuccess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1") t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success") t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json") recordPath := filepath.Join(t.TempDir(), "record.json")

View File

@@ -4,7 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// NoopRunner is a deterministic no-op seriatim adapter. // NoopRunner is a deterministic no-op seriatim adapter.
@@ -260,7 +260,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
func materializePlaceholders(req MergeRequest) error { func materializePlaceholders(req MergeRequest) error {
if req.OutputMergedTranscriptPath != "" { if req.OutputMergedTranscriptPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err) return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
} }
} }
@@ -271,22 +271,22 @@ func materializePlaceholders(req MergeRequest) error {
"input_transcript_paths": req.InputTranscriptPaths, "input_transcript_paths": req.InputTranscriptPaths,
"output_path": req.OutputMergedTranscriptPath, "output_path": req.OutputMergedTranscriptPath,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }
if req.ReportPath != "" { if req.ReportPath != "" {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write report %q: %w", req.ReportPath, err) return fmt.Errorf("write report %q: %w", req.ReportPath, err)
} }
} }
@@ -295,7 +295,7 @@ func materializePlaceholders(req MergeRequest) error {
func materializeTrimPlaceholders(req TrimRequest) error { func materializeTrimPlaceholders(req TrimRequest) error {
if req.OutputTrimmedPath != "" { if req.OutputTrimmedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err) return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
} }
} }
@@ -308,17 +308,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
"output_path": req.OutputTrimmedPath, "output_path": req.OutputTrimmedPath,
"keep_selector": req.KeepSelector, "keep_selector": req.KeepSelector,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }
@@ -327,7 +327,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
func materializeNormalizePlaceholders(req NormalizeRequest) error { func materializeNormalizePlaceholders(req NormalizeRequest) error {
if req.OutputNormalizedPath != "" { if req.OutputNormalizedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err) return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
} }
} }
@@ -343,22 +343,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
if req.ReportPath != "" { if req.ReportPath != "" {
payload["report_path"] = req.ReportPath payload["report_path"] = req.ReportPath
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }
if req.ReportPath != "" { if req.ReportPath != "" {
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write report %q: %w", req.ReportPath, err) return fmt.Errorf("write report %q: %w", req.ReportPath, err)
} }
} }
@@ -367,7 +367,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
func materializeRenderPlaceholders(req RenderRequest) error { func materializeRenderPlaceholders(req RenderRequest) error {
if req.OutputRenderedPath != "" { if req.OutputRenderedPath != "" {
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err) return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
} }
} }
@@ -384,17 +384,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
"include_segment_ids": req.IncludeSegmentIDs, "include_segment_ids": req.IncludeSegmentIDs,
"include_metadata": req.IncludeMetadata, "include_metadata": req.IncludeMetadata,
} }
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil { if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err) return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
} }
} }
if req.StdoutLogPath != "" { if req.StdoutLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err) return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
} }
} }
if req.StderrLogPath != "" { if req.StderrLogPath != "" {
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), 0o644); err != nil { if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err) return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
} }
} }

View File

@@ -0,0 +1,19 @@
package seriatim
import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gopkg.in/yaml.v3"
)
func writeGeneratedYAML(path string, payload any) error {
data, err := yaml.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
}
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write generated YAML %q: %w", path, err)
}
return nil
}

View File

@@ -4,15 +4,18 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"unicode/utf8" "unicode/utf8"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess" "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
) )
// MaxOutputFileBytes bounds each Seriatim JSON or rendered-text result.
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
// EnvConfig defines optional Seriatim environment tuning values. // EnvConfig defines optional Seriatim environment tuning values.
type EnvConfig struct { type EnvConfig struct {
OverlapWordRunGap *float64 OverlapWordRunGap *float64
@@ -129,12 +132,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req MergeRequest) (MergeResu
} }
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{ runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: r.binary, Executable: r.binary,
Args: args, Args: args,
Timeout: r.timeout, Timeout: r.timeout,
EnvOverrides: env, EnvOverrides: env,
StdoutLogPath: req.StdoutLogPath, DiagnosticOwner: "seriatim",
StderrLogPath: req.StderrLogPath, StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
if err != nil { if err != nil {
return MergeResult{ return MergeResult{
@@ -232,11 +236,12 @@ func (r *SubprocessRunner) Trim(ctx context.Context, req TrimRequest) (TrimResul
} }
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{ runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: binary, Executable: binary,
Args: args, Args: args,
Timeout: timeout, Timeout: timeout,
StdoutLogPath: req.StdoutLogPath, DiagnosticOwner: "seriatim",
StderrLogPath: req.StderrLogPath, StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
if err != nil { if err != nil {
return TrimResult{ return TrimResult{
@@ -320,11 +325,12 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
} }
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{ runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: binary, Executable: binary,
Args: args, Args: args,
Timeout: timeout, Timeout: timeout,
StdoutLogPath: req.StdoutLogPath, DiagnosticOwner: "seriatim",
StderrLogPath: req.StderrLogPath, StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
if err != nil { if err != nil {
return NormalizeResult{ return NormalizeResult{
@@ -425,11 +431,12 @@ func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (Rende
} }
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{ runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: binary, Executable: binary,
Args: args, Args: args,
Timeout: timeout, Timeout: timeout,
StdoutLogPath: req.StdoutLogPath, DiagnosticOwner: "seriatim",
StderrLogPath: req.StderrLogPath, StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
}) })
if err != nil { if err != nil {
return RenderResult{ return RenderResult{
@@ -546,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
payload["coalesce_gap"] = *r.coalesceGap payload["coalesce_gap"] = *r.coalesceGap
} }
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644) return writeGeneratedYAML(req.GeneratedConfigPath, payload)
} }
func buildTrimArgs(req TrimRequest) []string { func buildTrimArgs(req TrimRequest) []string {
@@ -598,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
"output_path": req.OutputTrimmedPath, "output_path": req.OutputTrimmedPath,
"keep_selector": req.KeepSelector, "keep_selector": req.KeepSelector,
} }
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644) return writeGeneratedYAML(req.GeneratedConfigPath, payload)
} }
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error { func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
@@ -613,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
"output_schema": outputSchema, "output_schema": outputSchema,
"report_path": req.ReportPath, "report_path": req.ReportPath,
} }
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644) return writeGeneratedYAML(req.GeneratedConfigPath, payload)
} }
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error { func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
@@ -631,13 +638,13 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
"include_segment_ids": req.IncludeSegmentIDs, "include_segment_ids": req.IncludeSegmentIDs,
"include_metadata": req.IncludeMetadata, "include_metadata": req.IncludeMetadata,
} }
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644) return writeGeneratedYAML(req.GeneratedConfigPath, payload)
} }
func validateJSONFile(path string) error { func validateJSONFile(path string) error {
data, err := os.ReadFile(path) data, err := readSeriatimResult(path, "JSON output")
if err != nil { if err != nil {
return fmt.Errorf("read file: %w", err) return err
} }
var v any var v any
if err := json.Unmarshal(data, &v); err != nil { if err := json.Unmarshal(data, &v); err != nil {
@@ -647,9 +654,9 @@ func validateJSONFile(path string) error {
} }
func validateJSONFileWithSegments(path string) error { func validateJSONFileWithSegments(path string) error {
data, err := os.ReadFile(path) data, err := readSeriatimResult(path, "transcript JSON output")
if err != nil { if err != nil {
return fmt.Errorf("read file: %w", err) return err
} }
var payload map[string]any var payload map[string]any
@@ -668,9 +675,9 @@ func validateJSONFileWithSegments(path string) error {
} }
func validateNonEmptyTextFile(path string) error { func validateNonEmptyTextFile(path string) error {
data, err := os.ReadFile(path) data, err := readSeriatimResult(path, "rendered text output")
if err != nil { if err != nil {
return fmt.Errorf("read file: %w", err) return err
} }
if len(data) == 0 { if len(data) == 0 {
return fmt.Errorf("file is empty") return fmt.Errorf("file is empty")
@@ -683,3 +690,11 @@ func validateNonEmptyTextFile(path string) error {
} }
return nil return nil
} }
func readSeriatimResult(path, category string) ([]byte, error) {
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
if err != nil {
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
}
return data, nil
}

View File

@@ -6,16 +6,12 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"strings" "strings"
"testing" "testing"
"time" "time"
) )
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) { func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "success") t.Setenv("SERIATIM_HELPER_MODE", "success")
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
} }
func TestSubprocessRunnerSubprocessFailure(t *testing.T) { func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail") t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
} }
func TestSubprocessRunnerMissingOutputFails(t *testing.T) { func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output") t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
} }
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) { func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output") t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) { func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report") t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) { func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "trim_success") t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
} }
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) { func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail") t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
} }
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) { func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output") t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
} }
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) { func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output") t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) { func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments") t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) { func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success") t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) { func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail") t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) { func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output") t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) { func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output") t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) { func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments") t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) { func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing") t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
} }
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) { func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report") t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
} }
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) { func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success") t.Setenv("SERIATIM_HELPER_MODE", "render_success")
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
} }
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) { func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success") t.Setenv("SERIATIM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json") recordPath := filepath.Join(t.TempDir(), "record.json")
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
} }
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) { func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail") t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
} }
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) { func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output") t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
} }
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) { func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1") t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output") t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json")) t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))

View File

@@ -0,0 +1,90 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
"math"
"strings"
)
// ReadLimitError reports that a remote object exceeded its caller-owned read
// limit. The limit is enforced against both available object metadata and the
// bytes returned by the opened object body.
type ReadLimitError struct {
Key string
Limit int64
Observed int64
}
func (e *ReadLimitError) Error() string {
return fmt.Sprintf("object %q exceeds %d-byte read limit (observed at least %d bytes)", e.Key, e.Limit, e.Observed)
}
// ReadObjectBounded opens one object version and retains at most maxBytes of
// its content. Object metadata may reject an oversized body early, but a
// limit-plus-one read always enforces the boundary when transfer begins.
func ReadObjectBounded(ctx context.Context, store ObjectStore, key string, maxBytes int64) (info ObjectInfo, data []byte, err error) {
key = strings.TrimSpace(key)
if store == nil {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: store is required")
}
if key == "" {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: key is required")
}
if maxBytes <= 0 || maxBytes == math.MaxInt64 {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: limit must be between 1 and %d bytes", key, int64(math.MaxInt64-1))
}
if err := ctx.Err(); err != nil {
return ObjectInfo{}, nil, err
}
info, body, err := store.Read(ctx, key)
if err != nil {
return ObjectInfo{}, nil, err
}
if body == nil {
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: store returned no body", key)
}
defer func() {
if closeErr := body.Close(); closeErr != nil {
data = nil
err = errors.Join(err, fmt.Errorf("close object %q: %w", key, closeErr))
}
}()
if info.Size > maxBytes {
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: info.Size}
}
data, err = io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, maxBytes+1))
if err != nil {
return info, nil, err
}
if err := ctx.Err(); err != nil {
return info, nil, err
}
if int64(len(data)) > maxBytes {
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: int64(len(data))}
}
return info, data, nil
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r contextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
n, err := r.reader.Read(p)
if err == nil {
if contextErr := r.ctx.Err(); contextErr != nil {
return n, contextErr
}
}
return n, err
}

View File

@@ -0,0 +1,135 @@
package storage
import (
"bytes"
"context"
"errors"
"io"
"testing"
)
func TestReadObjectBoundedAcceptsExactLimitWithAbsentSizeMetadata(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 2}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", ETag: "generation"}, body, nil
}}
info, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
if err != nil {
t.Fatalf("ReadObjectBounded() error = %v", err)
}
if string(data) != "12345678" || info.ETag != "generation" {
t.Fatalf("ReadObjectBounded() = (%#v, %q), want opened object metadata and bytes", info, data)
}
if !body.closed {
t.Fatal("object body was not closed")
}
}
func TestReadObjectBoundedRejectsLimitPlusOneDespiteMissingOrInaccurateMetadata(t *testing.T) {
tests := []struct {
name string
metadataSize int64
}{
{name: "missing", metadataSize: 0},
{name: "inaccurate", metadataSize: 2},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("123456789")), chunkSize: 1}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: test.metadataSize}, body, nil
}}
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
var limitErr *ReadLimitError
if !errors.As(err, &limitErr) {
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
}
if data != nil || body.bytesRead != 9 || !body.closed {
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 9, true", data, body.bytesRead, body.closed)
}
})
}
}
func TestReadObjectBoundedRejectsOversizedMetadataBeforeTransfer(t *testing.T) {
body := &trackingReadCloser{reader: bytes.NewReader([]byte("small"))}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: 9}, body, nil
}}
_, _, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
var limitErr *ReadLimitError
if !errors.As(err, &limitErr) {
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
}
if body.bytesRead != 0 || !body.closed {
t.Fatalf("bytes read=%d closed=%t, want zero-byte transfer and closed body", body.bytesRead, body.closed)
}
}
func TestReadObjectBoundedPropagatesCancellationAndClosesBody(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 1, afterRead: cancel}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json"}, body, nil
}}
_, data, err := ReadObjectBounded(ctx, store, "control.json", 8)
if !errors.Is(err, context.Canceled) {
t.Fatalf("ReadObjectBounded() error = %v, want context cancellation", err)
}
if data != nil || body.bytesRead != 1 || !body.closed {
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 1, true", data, body.bytesRead, body.closed)
}
}
func TestReadObjectBoundedReturnsCloseFailure(t *testing.T) {
closeErr := errors.New("close failed")
body := &trackingReadCloser{reader: bytes.NewReader([]byte("ok")), closeErr: closeErr}
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
return ObjectInfo{Key: "control.json", Size: 2}, body, nil
}}
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
if !errors.Is(err, closeErr) || data != nil || !body.closed {
t.Fatalf("data=%q error=%v closed=%t, want close failure and no retained data", data, err, body.closed)
}
}
type boundedReadStore struct {
ObjectStore
read func(context.Context, string) (ObjectInfo, io.ReadCloser, error)
}
func (s *boundedReadStore) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
return s.read(ctx, key)
}
type trackingReadCloser struct {
reader io.Reader
chunkSize int
afterRead func()
closeErr error
bytesRead int
closed bool
}
func (r *trackingReadCloser) Read(p []byte) (int, error) {
if r.chunkSize > 0 && len(p) > r.chunkSize {
p = p[:r.chunkSize]
}
n, err := r.reader.Read(p)
r.bytesRead += n
if n > 0 && r.afterRead != nil {
r.afterRead()
r.afterRead = nil
}
return n, err
}
func (r *trackingReadCloser) Close() error {
r.closed = true
return r.closeErr
}

View File

@@ -0,0 +1,23 @@
package storage
import (
"context"
"fmt"
"io"
)
// WriterDownloader is implemented by storage backends that stream an object
// into a caller-owned file handle.
type WriterDownloader interface {
DownloadTo(ctx context.Context, key string, destination io.Writer) error
}
// DownloadTo streams one object into destination. Destination-confined callers
// require this capability rather than granting a backend a mutable pathname.
func DownloadTo(ctx context.Context, store ObjectStore, key string, destination io.Writer) error {
writer, ok := store.(WriterDownloader)
if !ok {
return fmt.Errorf("object store does not support handle-confined downloads")
}
return writer.DownloadTo(ctx, key, destination)
}

View File

@@ -14,16 +14,15 @@ func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectSt
return nil, fmt.Errorf("pipeline config is required") return nil, fmt.Errorf("pipeline config is required")
} }
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") { switch strings.ToLower(strings.TrimSpace(cfg.Pipeline.Storage.Backend)) {
case config.StorageBackendS3:
if cfg.Pipeline.Storage.S3 == nil { if cfg.Pipeline.Storage.S3 == nil {
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3") return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
} }
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3) return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
case "", config.StorageBackendLocal:
return nil, fmt.Errorf("no remote object store backend is configured")
default:
return nil, fmt.Errorf("unsupported pipeline.storage.backend %q", cfg.Pipeline.Storage.Backend)
} }
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
}
return nil, fmt.Errorf("no remote object store backend is configured")
} }

View File

@@ -54,3 +54,35 @@ func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err) t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
} }
} }
func TestNewObjectStoreFromConfigDoesNotInferS3FromProviderFields(t *testing.T) {
called := false
original := newS3Client
t.Cleanup(func() { newS3Client = original })
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
called = true
return &fakeS3API{}, nil
}
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{
Backend: config.StorageBackendLocal,
S3: &config.StorageS3Config{Bucket: "my-archive"},
}},
})
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
}
if called {
t.Fatal("NewObjectStoreFromConfig() constructed S3 from incidental provider fields")
}
}
func TestNewObjectStoreFromConfigRejectsUnknownBackend(t *testing.T) {
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{Backend: "s33"}},
})
if err == nil || !strings.Contains(err.Error(), "unsupported pipeline.storage.backend") {
t.Fatalf("NewObjectStoreFromConfig() error = %v, want unsupported-backend error", err)
}
}

Some files were not shown because too many files have changed in this diff Show More