79 Commits

Author SHA1 Message Date
b750c42ad6 Remove legacy manifest provenance decoding
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
2026-09-25 19:45:19 +00:00
73c4986f3d Preserve Audita module selections in YAML output 2026-09-25 19:42:22 +00:00
e5c06019b3 Record migration review findings 2026-09-25 18:28:38 +00:00
fae18bf4a7 Verify shared provider credentials across both adapters 2026-09-25 16:06:02 +00:00
e5c1029165 Document Audita v2 operator migration and examples 2026-09-25 16:00:27 +00:00
2d167d0d67 Track Audita v2 selections in polish resume evidence 2026-09-25 15:55:38 +00:00
569f467393 Archive Audita reports as polish diagnostics 2026-09-25 15:51:18 +00:00
77d9d57937 Adopt Audita v2 configuration and subprocess contract 2026-09-25 15:45:02 +00:00
7e0151ef93 Migrate Notarius to credential lists 2026-09-25 15:33:46 +00:00
49948852d6 Add shared resolved credential forwarding 2026-09-25 15:26:51 +00:00
83eaa960ec Adopt ConfigKit overlays for pipeline selection 2026-09-25 15:24:03 +00:00
e27c49e946 Track repository agent instructions 2026-09-25 15:11:56 +00:00
625e6edfd7 Align repository policies with shared standards 2026-09-25 15:11:08 +00:00
17cef0700a Add ConfigKit overlay migration to the plan 2026-09-25 15:04:55 +00:00
55c102c9ed Plan the Audita v2 migration 2026-09-25 14:50:25 +00:00
70f1dbfb59 Prepare the v2.2.1 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-22 18:18:45 +00:00
f1575effc2 Document shared checkpoint persistence 2026-09-22 15:13:46 +00:00
7e39a1d51f Delegate transcription checkpoints to shared persistence 2026-09-22 15:12:11 +00:00
3722e572df Delegate analyze checkpoints to shared persistence 2026-09-22 15:09:29 +00:00
7147e95252 Add shared stage checkpoint persistence 2026-09-22 15:05:25 +00:00
9f2cfddce6 Plan shared stage checkpoint mechanics 2026-09-22 14:54:34 +00:00
792cbd0677 Prepare the v2.2.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-22 01:50:49 +00:00
5dca34e0e0 Allow checkpoints for empty audio accepted by WhisperX 2026-09-22 00:53:28 +00:00
54d335c820 Repair stale transcription outputs when resuming 2026-09-22 00:50:51 +00:00
96b4e73f53 Document resumable transcription and replay 2026-09-22 00:31:32 +00:00
6e6054f744 Verify WhisperX transport replay 2026-09-22 00:22:48 +00:00
dc3288d15a Make WhisperX uploads replayable 2026-09-22 00:15:48 +00:00
d872edca94 Integrate transcription checkpoints across lifecycle 2026-09-22 00:00:51 +00:00
2ae63dbc2b Checkpoint speaker transcriptions incrementally 2026-09-21 16:20:53 +00:00
26aa0cce29 Reconcile reusable speaker transcriptions 2026-09-21 16:09:45 +00:00
7b0bf99ed1 Add durable speaker transcription state 2026-09-21 16:01:19 +00:00
89ada1b269 Plan resumable transcription and replayable uploads 2026-09-21 15:50:48 +00:00
3438fe82c9 Document Notarius configuration profiles 2026-09-21 15:16:11 +00:00
a51a513804 Verify Notarius profile lifecycle behavior 2026-09-21 15:11:19 +00:00
995354f1a5 Track Notarius profiles in extraction reuse 2026-09-21 15:07:36 +00:00
931c9591cd Add Notarius configuration profile support 2026-09-21 15:03:29 +00:00
46096fc4ba Plan Notarius configuration profile support 2026-09-21 14:54:47 +00:00
d3ecd907ee Update ConfigKit and prepare Narratio v2.1.2
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-21 13:42:21 +00:00
442469b9f5 Fix derived publish destinations with explicit requiredness
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-21 03:06:13 +00:00
c45e042462 Prepare the v2.1.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-16 18:50:30 +00:00
207da1b6ef Remove completed Notarius cache roadmaps
Some checks failed
ci/woodpecker/push/verify Pipeline was canceled
2026-09-16 18:48:33 +00:00
9a1609435e Clarify Notarius compatibility requirements 2026-09-16 18:48:23 +00:00
13b83d7ee2 Fix Notarius snapshots in relative workspaces 2026-09-16 18:48:23 +00:00
a0312ed034 Document Notarius cache reuse workflow 2026-09-16 18:14:40 +00:00
8ff74e1251 Prove Notarius cache reuse end to end 2026-09-16 18:07:37 +00:00
1b480d4ed5 Verify Notarius cache lifecycle boundaries 2026-09-16 17:59:52 +00:00
e13d87a077 Use stable verified Notarius reference snapshots 2026-09-16 17:53:29 +00:00
5202f16972 Forward Notarius cache controls during extraction 2026-09-16 17:44:28 +00:00
5086596f94 Add typed Notarius cache controls 2026-09-16 17:42:16 +00:00
f7fc64db10 Plan Notarius cache reuse 2026-09-16 17:34:21 +00:00
6b0778b420 Clean up completed roadmap documentation
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
2026-09-16 16:39:37 +00:00
5b92b3e790 Document Notarius credential forwarding 2026-09-16 14:36:44 +00:00
cc43906ae9 Verify Notarius credential lifecycle 2026-09-16 14:31:54 +00:00
dcaa2ad9c6 Forward Notarius credentials securely 2026-09-16 14:27:06 +00:00
9d68cfcf2f Add Notarius credential configuration 2026-09-16 14:22:45 +00:00
2d346733d6 Plan Notarius credential forwarding 2026-09-16 14:14:56 +00:00
cd75bf6d5a Complete the ConfigKit v0.3.0 adoption 2026-09-16 04:33:19 +00:00
e08de98ae6 Upgrade ConfigKit to v0.3.0 2026-09-16 04:29:05 +00:00
df213bc2ba Isolate pipeline configuration discovery 2026-09-16 04:26:24 +00:00
869e68e834 Plan the ConfigKit upgrade and test isolation 2026-09-16 04:16:30 +00:00
ed4623bc41 Fix profile loading with ConfigKit history
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-07 13:10:29 +00:00
3ce198197d Report secrets bootstrap outcomes safely 2026-09-07 12:14:10 +00:00
0ac8072a9a Complete artifact provenance reporting 2026-09-07 12:08:19 +00:00
3ec54b8c72 Preserve null collections in effective YAML 2026-09-07 12:03:45 +00:00
47b3e5b55b Validate hidden base artifact ownership conflicts 2026-09-07 11:59:49 +00:00
99d1a0b926 Honor explicit Notarius working directories 2026-09-07 11:54:35 +00:00
884a62e8a6 Derive family member publish destinations 2026-09-07 11:52:57 +00:00
f7a40afc7a Complete the ConfigKit migration audit 2026-09-07 04:58:52 +00:00
a4f37ff427 Document the v2 operator migration 2026-09-07 04:55:13 +00:00
2b97ef3722 Add effective artifact catalog command 2026-09-07 04:46:20 +00:00
63a86c43f4 Infer artifact dependencies from configured inputs 2026-09-07 04:39:49 +00:00
92432232ef Resolve typed credentials safely 2026-09-07 04:29:00 +00:00
d31ba5a410 Bootstrap secrets before configuration loading 2026-09-07 04:14:04 +00:00
f0225e937c Migrate remaining configuration documents to ConfigKit 2026-09-07 04:02:21 +00:00
9e40c729e1 Adopt ConfigKit for pipeline loading 2026-09-07 03:54:41 +00:00
0e6ba414b3 Unify effective config projection 2026-09-07 03:35:46 +00:00
10a2e89965 Separate config normalization from validation 2026-09-07 03:32:50 +00:00
f8e3e651e6 Reduce supported platform matrix 2026-09-07 03:28:50 +00:00
1558c9ccae Plan ConfigKit adoption and platform simplification 2026-09-07 03:21:06 +00:00
211 changed files with 15160 additions and 7216 deletions

2
.gitignore vendored
View File

@@ -1,6 +1,5 @@
# ---> Codex
.codex
AGENTS.md
.DS_Store
@@ -52,4 +51,3 @@ pipeline.yml
# Built Visual Studio Code Extensions
*.vsix

View File

@@ -40,7 +40,7 @@ steps:
set -eu
output_dir="$(mktemp -d)"
trap 'rm -rf "$output_dir"' EXIT
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
for target in linux/amd64 linux/arm64 darwin/arm64; do
goos="${target%/*}"
goarch="${target#*/}"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio

5
AGENTS.md Normal file
View File

@@ -0,0 +1,5 @@
Please review `docs/development.md` for initial orientation in this repository
and follow its task-specific reading guide.
Review and follow `docs/policy/agents.md` before recommending, planning, or
implementing any changes to this repository.

View File

@@ -20,7 +20,7 @@ Top-level commands:
- `publish <session_id>`: force-run publish.
- `clean <session_id>` or `clean --all`: remove local work/spool state.
- `session <subcommand>`: session helper commands.
- `config <subcommand>`: validate, display, source-trace, or compare resolved pipeline configuration.
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
Session subcommands:
@@ -44,7 +44,8 @@ Most session-aware commands accept:
- `--session <session.yml>`
- `--session-id <session_id>`
- `--previous-session-id <session_id>`
- `--profile <name>`
- `--overlay <name>`
- `--secrets-dir <directory>`
Rules:
@@ -53,10 +54,13 @@ Rules:
- if both positional `<session_id>` and `--session-id` are provided, values must match.
- `--previous-session-id` is a strict expectation: the selected session file
must contain the same `previous_session_id`.
- `--profile` selects a declared pipeline profile. It may be supplied once;
an explicit empty or unknown value fails configuration resolution. When it is
omitted, a declared `default_profile` is used. The same selection applies to
all common-flag commands, including `regenerate-artifacts`.
- `--overlay` selects `overlays/<name>.yml` relative to the pipeline root. It
may be supplied once; omitted, empty, and the exact name `default` select the
base configuration, while an unknown named overlay fails. The same selection
applies to all common-flag commands, including `regenerate-artifacts`.
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
relative value uses caller CWD. Without the flag, a present `secrets/`
directory beside the resolved root pipeline is used; an absent one is skipped.
- `clean --all` cannot be combined with campaign/session selectors.
- notification delivery is currently limited to the configured `noop` mode; see
the [configuration reference](./config.md#notifications).
@@ -78,19 +82,24 @@ Commands with additional positionals keep their command-specific order:
## Command Reference
### `config validate`, `config show`, `config sources`, and `config diff`
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
```bash
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--overlay <name>] [--secrets-dir <directory>]
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--overlay <name>] [--secrets-dir <directory>]
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--overlay <name>] [--secrets-dir <directory>]
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--overlay <name>] [--secrets-dir <directory>]
narratio config diff <left-overlay> <right-overlay> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
```
These commands resolve the selected profile, defaults, ordinary paths, and—if
These commands resolve the selected overlay, defaults, ordinary paths, and—if
a campaign is selected—the campaign-owned party. They neither discover or load
a session nor create a workspace, manifest, run, lock, adapter, remote
connection, or credential environment.
a session nor create a workspace, manifest, run, lock, adapter, or remote
connection. Like every pipeline-loading command, they bootstrap the selected or
conventional secrets directory before resolving configuration; `config show`
never includes bootstrap reports or secret values in its YAML output.
Typed credential references, including `notarius.credentials[]`, must therefore
resolve even though these inspection commands never construct an adapter.
Campaign selection is optional for a pipeline without party-driven artifact
families. A pipeline with `scriptorium.artifact_families` needs a selected or
@@ -98,29 +107,40 @@ configured default campaign so Narratio can expand its concrete artifacts and
publish rules. `--campaign` and `--campaign-file` remain mutually exclusive.
Session, range, force, and artifact-execution flags are not accepted.
`config validate` writes a concise root-path, selected-profile (or `none`), and
`config validate` writes a concise root-path, selected-overlay (or `none`), and
effective-digest summary after successful complete validation. `config show`
writes one deterministic, secret-free YAML document containing defaulted and
expanded concrete configuration. It omits composition declarations, artifact
expanded concrete configuration. It omits ConfigKit directives, artifact
family declarations, and runtime provenance.
`config sources` reports the same fully validated resolution without printing
effective values. Its header identifies the root, ordered imports, selected
profile and overlay, selected campaign, party source, and digest. The
overlay and its file, selected campaign, party source, and digest. The
remaining tab-separated records are sorted as `path`, `role`, and `source`.
Roles distinguish root, import, profile, centralized default, campaign, party,
Roles distinguish root, import, overlay, centralized default, campaign, party,
and generated family ownership. A generated party member has
one family record and one party record at the same logical path. The output
never reads or prints secret values.
`config diff` resolves both supplied profile names from one parsed root source
set and compares their fully resolved, secret-free effective mappings. It does
not accept `--profile`; the two positional names must be distinct, declared
profiles. When party-driven families are present, both profiles must resolve to
the same selected campaign and party. Use `--campaign-file` if profile-specific
`config artifacts` prints the concrete catalog used by analysis, ordered by
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
`party_source`. Definitions and effective dependencies are sorted comma-separated
lists, and control characters in fields are escaped. Family-member rows identify
their family, character, defining family file, and party file. An empty catalog
prints only the header. Ordinary artifacts can be inspected without a campaign;
party-driven families retain the campaign requirement. See
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
configuration layout.
`config diff` resolves both supplied overlay names independently and compares
their fully resolved, secret-free effective mappings. It does
not accept `--overlay`; the two positional names must be distinct, valid
overlays. When party-driven families are present, both overlays must resolve to
the same selected campaign and party. Use `--campaign-file` if overlay-specific
campaign configuration would otherwise select different files.
Equal profiles print `no differences`. Otherwise, sorted tab-separated records
Equal overlays print `no differences`. Otherwise, sorted tab-separated records
use one of these forms, with compact JSON values:
```text
@@ -131,7 +151,7 @@ changed <path> <left-value> <right-value>
Mappings are flattened to their logical field paths; lists remain one atomic
value. The command compares defaulted concrete artifacts and publish rules, not
profile names, source-file layout, or formatting. It succeeds when differences
overlay names, source-file layout, or formatting. It succeeds when differences
are found, making it suitable for review and migration checks.
### `version`
@@ -157,6 +177,9 @@ Behavior:
- rejects unknown endpoints and a `--from` endpoint after `--through`;
- runs `render` before `extract`; an omitted or disabled Notarius
configuration records an explicit `notarius_disabled` self-skip;
- resolves every selected typed credential during the initial configuration
load, including Notarius credentials when extract is disabled, outside the
selected range, or later skipped as resumable;
- skips already-succeeded stages unless `--force` is set or a stage-specific
resume check finds its durable result obsolete;
- applies `--force` only to stages in the selected range;
@@ -166,7 +189,7 @@ Behavior:
- writes the session manifest as the sole durable progress ledger. Each
invocation also creates its run-local directory and returns an immediate
command summary; neither creates another durable progress record.
- reports the resolved profile (or `none`) and effective configuration digest.
- reports the resolved overlay (or `none`) and effective configuration digest.
When `--artifacts` is present, the selected range must contain `analyze` or
`publish`. Either consumer is sufficient, including a one-stage range.
@@ -189,6 +212,10 @@ artifacts and any prerequisites required by those targets; without
never run. Common session/configuration options and repeatable artifact values
pass through unchanged.
The forced extraction rebuilds the complete configured Notarius bundle,
disables Notarius checkpoint loading, and requests chunk-plan refresh.
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
supply those options. The shared `run` parser reports them as duplicate
singleton flags. The alias has no private execution options or behavior, and
@@ -231,6 +258,10 @@ Equivalent to:
narratio run-stage analyze <session_id> --force [...common config flags]
```
This selects only `analyze`: it does not invoke Notarius, replace extraction,
or remove existing extract outputs. `--artifacts` narrows the forced analysis
targets and their required analysis prerequisites.
### `publish`
```bash
@@ -273,7 +304,7 @@ Uses the same inclusive bounds, endpoint validation, force scope, and artifact
selection contract as `run`. It validates config and prints run/skip decisions
for selected stages only without creating the local workdir or changing the
manifest. Resume-capable selected stages are checked against durable evidence.
The output includes the resolved profile (or `none`) and effective configuration
The output includes the resolved overlay (or `none`) and effective configuration
digest without writing provenance or any manifest state.
For `analyze`, the preview also lists explicit targets, prerequisite-only work,
execution order, and reusable current artifacts with concise reasons. These
@@ -320,6 +351,7 @@ Options:
- `--config <pipeline.yml>`
- `--campaign <id>` or `--campaign-file <campaign.yml>`
- `--secrets-dir <directory>`
- `--previous-session-id <id>`
- `--date <YYYY-MM-DD>`
- `--title <text>`

View File

@@ -44,25 +44,30 @@ the durable copied session input.
### Read-only effective pipeline inspection
`narratio config validate`, `narratio config show`, and `narratio config
sources` use the same `--config`, `--campaign`, `--campaign-file`, and
`--profile` selection rules as pipeline commands, but do not select, discover,
or load a session. They do not read credential values or create runtime state.
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
campaign selectors, resolves each named profile independently from one parsed
root source set, and does not accept a separate `--profile` flag.
`narratio config validate`, `narratio config show`, `narratio config sources`,
and `narratio config artifacts` use the same `--config`, `--campaign`,
`--campaign-file`, and
`--overlay` selection rules as pipeline commands, but do not select, discover,
or load a session. They resolve configured typed credential references as part
of the ordinary ConfigKit load but expose only their environment-variable names;
they do not create runtime state.
`narratio config diff <left-overlay> <right-overlay>` uses the same pipeline and
campaign selectors, resolves each named overlay with an independent ConfigKit
load, and does not accept a separate `--overlay` flag.
Campaign selection is optional only when the resolved pipeline has no
`scriptorium.artifact_families`. When families are declared, Narratio selects a
campaign through an explicit flag or `pipeline.campaigns.default_campaign_id`,
then parses the campaign-owned party and expands concrete artifacts and any
family publish rules before validation. `config validate` prints the resulting
root, profile, and effective digest. `config show` emits the normalized
root, overlay, and effective digest. `config show` emits the normalized
effective pipeline YAML, with defaults and concrete expansion included but
composition and family declarations omitted. `config sources` prints a stable
source projection instead of effective values: root/import/profile/default
ConfigKit directives and family declarations omitted. This review projection is
not promised to be a reloadable authoring document. `config sources` prints a stable
source projection instead of effective values: root/import/overlay/default
ownership plus campaign/party and generated-family records. Canonical derived
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
players trace to the party. `config artifacts` lists the concrete artifact map,
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
owns command syntax and output conventions.
`config diff` compares normalized field values rather than YAML text or source
@@ -76,13 +81,14 @@ difference.
### Migrating to the maintained bundle
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
as the complete copyable migration reference. Split stable pipeline settings
into explicit additive imports, place production/testing differences in one
selected overlay, and retain a production `default_profile`. Convert campaign
rosters to [canonical party input](integrations/party.md), then express
character work as families. Inspect the result
with `config validate`, `config show`, and `config sources`; use `config diff`
to review profiles before running a session. Unversioned party documents are
as the complete copyable migration reference. Keep production settings in the
base, split them into explicit additive imports, and place testing differences
in the fixed `overlays/testing.yml` overlay. Put ordinary artifacts and families
in the recommended one-definition files shown by that bundle. Convert campaign
rosters to [canonical party input](integrations/party.md), then express character
work as families. Inspect the result with `config validate`, `config show`,
`config sources`, and `config artifacts`; use `config diff default testing` to
review the overlay before running a session. Unversioned party documents are
rejected.
### Identity segments
@@ -96,20 +102,31 @@ remote state with an unsafe legacy identity must be migrated before use.
## Validation and Merge Rules
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
unknown fields or trailing documents fail load.
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
unknown fields, duplicate keys, and trailing documents fail load.
- A pipeline file may explicitly import additive YAML fragments through the
root-only `composition.imports` list. Imported files contribute fields to one
root-only `configkit.imports` list. Imported files contribute fields to one
logical pipeline document; they do not override fields supplied by the root
or another import.
- A root pipeline may declare named profiles. Exactly one profile is selected
by an option-aware caller or by `composition.default_profile`; a caller's
explicit selection takes precedence. Declaring profiles without either form
of selection is an error.
- An omitted overlay, an empty overlay option, and the exact name `default`
select the base configuration. Any other name selects
`overlays/<name>.yml` relative to the pipeline root. Move previously selected
`profiles/<name>.yml` files there; the old directory is not searched, and
`--profile` is no longer accepted for Narratio pipeline selection. Explicit
imports from a directory named `profiles` remain valid.
- Campaign, session, and canonical-party documents are strict single-source
ConfigKit schemas. They do not enable imports, overlays, or `configkit`
directives.
- Configured timeout and retry-delay durations must be positive. An omitted
artifact timeout continues to inherit its configured Scriptorium timeout.
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
- Session and party inputs inspected before typed loading are limited to 8 MiB.
- Pipeline defaults are applied before validation.
- Defaults apply only to absent fields. Explicit empty strings, zero values,
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
Narratio domain validation; they are not silently replaced by defaults.
Explicit null optional stage sections stay absent, while required sections
fail validation.
- Campaign and session identities must agree.
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
`party_file`) are campaign-owned; the first three may be overridden by the
@@ -123,22 +140,16 @@ remote state with an unsafe legacy identity must be migrated before use.
- local (`audio_dir` or `audio_files`), or
- S3 (`audio_s3.prefix`).
### Pipeline composition
### Pipeline imports and overlays
Large pipeline configurations may be split into explicitly named fragments and
may declare one overlay per selectable profile:
may use fixed-path selected overlays:
```yaml
composition:
configkit:
imports:
- config/storage.yml
- config/integrations.yaml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
- path: config/storage.yml
- path: config/integrations.yaml
campaigns:
root: /usr/local/share/narratio/campaigns
@@ -152,37 +163,30 @@ Imports are resolved relative to the directory containing the root pipeline
file and are loaded in declaration order. Narratio does not scan directories or
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
absolute paths, traversal, symlinks, directories, duplicate files, and an
import of the root pipeline itself are rejected. Only the root pipeline may
contain `composition`; nested composition is rejected.
import of the root pipeline itself are rejected. Imported files cannot declare
another `configkit` control section.
Composition is additive. A map may be extended by multiple files when every
Base merging is additive. A map may be extended by multiple files when every
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
more than once, even when the repeated values are identical. Conflict errors
name the full field path and every source that claimed it. The assembled YAML is
then decoded against the normal strict pipeline schema and defaults are applied
once.
name the full field path and every source that claimed it. Narratio additionally
rejects an artifact or artifact-family key declared by more than one base file,
even when those files contribute disjoint children.
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
control characters. If `profiles` is present, it must contain at least one
entry and every entry must contain only an `overlay` path. An explicit profile
selection overrides `default_profile`; unknown and explicitly empty selections
fail. Narratio never selects the first profile implicitly and does not read a
profile selection from the environment.
Overlay names are case-sensitive and use ConfigKit's safe fixed-path naming
rules. Only the selected overlay is opened; a malformed unselected overlay does
not affect a base or different-overlay load. Maps merge recursively, overlay
scalars replace base scalars, and overlay lists replace base lists completely.
Explicit `false`, zero, empty strings, empty containers, and null values remain
explicit inputs and are not refilled by defaults. Overlays cannot inherit,
stack, import files, or declare another ConfigKit control section.
Every declared overlay is resolved relative to the root pipeline directory and
must satisfy the same confined regular-YAML-file rules as an import. Narratio
parses every declared overlay even when it is not selected, then applies only
the selected one. Maps merge recursively, overlay scalars replace base scalars,
and overlay lists replace base lists completely. Explicit `false`, zero, empty
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
kind changes are rejected. Profiles cannot inherit from or stack with other
profiles, and overlays cannot import files or declare profiles.
After composition, Narratio strictly decodes the result, applies centralized
defaults once, resolves ordinary paths, and computes a deterministic effective
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
entry-level defaults and normalization, read-only file-local validation, and
ordinary path resolution before computing a deterministic effective
configuration digest. The digest represents the normalized, secret-free
runtime pipeline mapping; it excludes composition declarations, source
provenance, profile identity, and raw environment secret values. Equivalent
runtime pipeline mapping; it excludes ConfigKit directives, source
provenance, overlay identity, and raw environment secret values. Equivalent
effective mappings therefore have the same digest regardless of how fields are
split among the root and imports.
@@ -224,15 +228,31 @@ inputs:
## Secrets Handling
- Do not place raw secrets in YAML.
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
environment-variable filename supplies one value; trailing CR/LF is removed.
- Credential fields contain environment-variable names, represented internally
as ConfigKit typed secret references (for example
`pipeline.notarius.credentials: [NOTARIUS_LLM_API_KEY]`). A configured reference
must resolve to a nonempty environment value during load.
- Every surviving reference is resolved during ordinary pipeline loading,
regardless of whether its integration is disabled, outside the selected
stage range, or eligible for resume. Omit the field or set it to `null` when
that deployment should not require the credential.
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
flag is omitted, Narratio uses a present `secrets/` directory beside the
resolved root `pipeline.yml`; an absent conventional directory is ignored.
Explicit relative paths are resolved from the caller's working directory,
and a missing or empty explicit selection is an error.
- Each immediate regular file whose name is a valid environment-variable name
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
selected directory, rejects non-hidden symlink entries, and limits input to
1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
files, directories, and symlinks) are ignored and omitted from bootstrap
reports, but still count toward the directory entry limit.
- An existing process environment value takes precedence over a credential file.
- Credential directories and files must not be symlinks and must be regular,
bounded files (at most 8 KiB per value). On POSIX, provision the directory
with no group/other access (normally `0700`) and files with no group/other
access (normally `0600`).
- Commands that need storage/auth load filesystem secrets before constructing adapters.
- On POSIX, operators must provision the directory with no group/other access
(normally `0700`) and files with no group/other access (normally `0600`);
ConfigKit validates type and bounds but does not enforce permission modes.
- Bootstrap happens once per command, before any configuration load or adapter
construction. Safe reports contain filenames, paths, and statuses, never values.
## Publish Configuration Summary
@@ -274,22 +294,19 @@ Rules:
| Field | Type | Required | Default / Rule |
| --- | --- | --- | --- |
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
| `pipeline.secrets.env_dir` | string | No | empty |
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
| `pipeline.storage.s3.region` | string | No | empty |
| `pipeline.storage.s3.endpoint` | string | No | empty |
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
@@ -317,19 +334,14 @@ Rules:
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
| `pipeline.audita.binary` | string | No | `audita` |
| `pipeline.audita.timeout` | duration | No | `3h` |
| `pipeline.audita.llm_api_key_env` | string | No | empty |
| `pipeline.audita.modules[]` | list[string] | No | empty |
| `pipeline.audita.base_url` | string | No | empty |
| `pipeline.audita.model` | string | No | empty |
| `pipeline.audita.total_llm_concurrency` | int | No | unset |
| `pipeline.audita.proposal_llm_concurrency` | int | No | unset |
| `pipeline.audita.validation_model` | string | No | empty |
| `pipeline.audita.validation_llm_concurrency` | int | No | unset |
| `pipeline.audita.credentials[]` | list[secret reference] | No | empty; exact environment-variable names forwarded to Audita |
| `pipeline.audita.modules[]` | list[string] | No | unset; omit to use Audita defaults, set `[]` for no modules |
| `pipeline.audita.overlay` | string | No | empty; no downstream overlay override |
| `pipeline.audita.proposal_profile` | string | No | empty; no downstream proposal-profile override |
| `pipeline.audita.validation_profile` | string | No | empty; no downstream validation-profile override |
| `pipeline.audita.transcript_description` | string | No | empty |
| `pipeline.audita.config_path` | string | No | empty |
| `pipeline.audita.output_schema` | string | No | empty |
| `pipeline.audita.work_dir_retention` | string | No | empty |
| `pipeline.audita.report` | bool | No | `true` |
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
| `pipeline.normalize.report` | bool | No | `true` |
@@ -345,10 +357,14 @@ Rules:
| `pipeline.trim.seriatim.report` | bool | No | `false` |
| `pipeline.notarius.enabled` | bool | No | `false` |
| `pipeline.notarius.binary` | string | No | `notarius` |
| `pipeline.notarius.credentials[]` | list[secret reference] | No | empty; exact environment-variable names forwarded to Notarius |
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
| `pipeline.render.enabled` | bool | No | `true` |
@@ -365,6 +381,100 @@ Rules:
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
### Audita Selection and Credentials
`pipeline.audita.config_path` optionally passes an Audita v2 configuration file;
when empty, Audita uses its own native config discovery. `overlay`,
`proposal_profile`, and `validation_profile` are independent, optional Audita
selectors. Empty or omitted strings add no command-line override. Narratio's
selected pipeline overlay is never inferred as an Audita overlay, and Narratio
does not create or validate Audita's backend or PromptKit profile files. Prepare
them with [Audita's v2 migration guide](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/migrations/v2.md).
Omit `modules` or set it to `null` to use Audita's configured default sequence.
Set `modules: []` to run no correction modules. A nonempty list is passed in
order, including repetitions. Narratio does not add a reporting or debug flag;
Audita's external configuration controls those settings.
`pipeline.audita.credentials` lists secret-reference names, not values. Narratio
resolves every listed name during pipeline loading and forwards the captured
nonempty value under that exact name to Audita. It does not inspect Audita's
configuration to discover names or map them to aliases. Audita v2 resolves
credentials for **all configured backends**, even ones not selected by the
current profiles: list every named backend credential in the selected Audita
configuration. An unauthenticated backend needs no reference. Omission, `null`,
or `[]` forwards nothing. A Narratio overlay replaces the entire list, and
`[]` clears it. See the [Audita integration contract](./integrations/audita.md)
for the subprocess and redaction boundary.
### Notarius Configuration Profile
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
configuration profile when extraction executes:
```yaml
notarius:
config_profile: downstream-economy
```
When the field is omitted or `null`, Narratio supplies no downstream profile
selector and preserves Notarius's existing configuration-selection behavior.
When present, Narratio forwards the exact string to Notarius, including an
empty string or `default`, which explicitly select Notarius's composed base
configuration. Notarius owns selector interpretation and validation; Narratio
does not restrict profile names, resolve profile files, or rewrite the value.
This selection is independent of Narratio's own `--overlay` selection. A
Narratio overlay may explicitly select a differently named Notarius overlay,
but Narratio never forwards its own overlay name automatically. It is also
independent of `pipeline.notarius.credentials`: an overlay may change either or
both fields according to the deployed Notarius configuration. See the
[Notarius integration contract](./integrations/notarius.md) for invocation and
failure behavior.
### Notarius Credentials
`pipeline.notarius.credentials` optionally lists environment-variable names
to forward to the Notarius child process:
```yaml
notarius:
credentials: [OPENROUTER_API_KEY, ANTHROPIC_API_KEY]
```
The list contains variable names, not credential values. Those names must
match the references used by the Notarius/PromptKit profiles that Notarius
loads. Notarius owns profile lookup; Narratio neither infers those names nor
remaps them. Omission, `null`, or `[]` forwards nothing. Each selected reference
must resolve at pipeline load time through the process environment or existing
secrets-directory bootstrap, even when Notarius is disabled, extract is outside
the requested range, or a completed extraction can be reused.
The Notarius child receives only Narratio's minimal execution environment plus
these exact selected names and loaded values. Other credentials are not forwarded.
Changing a value, name, presence, or absence does not invalidate extract's
semantic resume evidence. The public names remain part of safe effective
inspection, so renaming them may appear in `config diff` and change the effective
configuration digest; rotating only their values does neither. An overlay
replaces the whole list, including with `[]` to clear it.
See the [Notarius integration contract](./integrations/notarius.md) for the
delivery and redaction boundary.
### Notarius Cache Controls
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
extraction passes `--resume`; the deployed Notarius configuration must already
enable checkpoint recording and should select an explicit persistent checkpoint
root. Narratio does not inspect or generate that configuration.
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
configured mode in control. A nonempty value must be `auto`, `bypass`, or
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
Neither field changes Narratio's semantic extraction fingerprint, so changing
them alone does not rerun an otherwise complete durable extraction.
### Notarius Reference Bindings
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
@@ -394,8 +504,8 @@ It validates only selector structure and the prepared source vocabulary;
Notarius owns target-slot declarations and media compatibility.
Before extraction, Narratio resolves every binding from the current prepared
session manifest and streams it into a verified invocation-local snapshot whose
absolute path is passed to Notarius. Missing, unsafe, empty,
session manifest and streams it into a verified, content-addressed session
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
changed-during-copy, or checksum-inconsistent prepared evidence fails with
guidance to force `prepare`. Bindings are sorted by normalized selector and are
part of extraction fingerprint and resume identity. See the
@@ -430,7 +540,7 @@ For each `pipeline.scriptorium.artifacts.<name>`:
| Field | Type | Required | Rule |
| --- | --- | --- | --- |
| `enabled` | bool | No | `false` if omitted |
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; configured graph must be acyclic |
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
| `render_debug` | bool | No | per-artifact override |
| `prompt_id` | string | Conditional | required when artifact is enabled |
| `profile_id` | string | No | empty |
@@ -444,7 +554,7 @@ Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium re
Without `--artifacts`, analyze executes enabled configured artifacts. With an
explicit `--artifacts` list, the exact named configured artifacts are the
one-invocation targets even if their `enabled` values are false. Analyze closes
those targets over `depends_on`: a current prerequisite is reused, while a
those targets over the effective dependency graph: a current prerequisite is reused, while a
stale, missing, or failed prerequisite is rebuilt before its dependent.
Unrelated artifacts are not executed. Named targets and any prerequisite that
may require rebuilding must therefore have valid executable fields. This
@@ -462,6 +572,15 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
loading. Use the canonical `source` identifier to select the input; Narratio
does not provide adapter-specific input passthrough fields.
Each `narratio.artifact.<name>` input source automatically contributes an
effective dependency on that configured artifact. Narratio unions these inferred
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
result during final configuration resolution. Authors use `depends_on` only for
ordering constraints that are not already expressed by an artifact input.
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
to inspect keys, definition sources, outputs, and effective dependencies without
loading a session or creating runtime state.
### Scriptorium Artifact Families
`pipeline.scriptorium.artifact_families` declares a shared artifact template
@@ -493,9 +612,10 @@ Their resolved values are strings. A member variable may not reuse a static
other Scriptorium artifact.
Within a family only, an input source may use
`narratio.member_artifact.<family>`. The referenced family must be named in
that family's `member_dependencies`; resolution rewrites the source to the
corresponding ordinary `narratio.artifact.<family>_<character_id>` source.
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
infers the concrete member dependency. Use `member_dependencies` only for
additional same-character ordering constraints that are not already inputs.
This syntax is rejected in explicit artifacts and never reaches runtime stages
or Scriptorium.

View File

@@ -10,6 +10,10 @@ polished transcripts and generated artifacts. Start with the
[Architecture](policy/architecture.md) for normative system boundaries, and the
[Internal Overview](internal/overview.md) for implemented component ownership.
Agents must read [Agent Policy](policy/agents.md) before recommending, planning,
or implementing changes. It owns implementation discipline and the default
commit-after-completed-work rule; it does not authorize pushing or releasing.
## Go Prerequisite
Maintained development and verification require Go 1.26. Use the official Go
@@ -30,7 +34,7 @@ Maintained development and verification require Go 1.26. Use the official Go
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
| Preparing, validating, or publishing a release | [Release Procedure](release.md) | The maintainer procedure owns version selection, candidate validation, guarded tag publication, and optional later CI inspection. |
| Preparing, validating, or publishing a release | [Release Policy](policy/release.md) and [Release Procedure](release.md) | Policy owns requirements and exceptions; the procedure owns concrete validation, guarded publication, and optional later inspection. |
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and package-level
@@ -58,7 +62,7 @@ in the [Release Procedure](release.md).
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
job to expose ordering and repeatability defects. Current runners cross-compile
for macOS and Windows, but do not provide native macOS or Windows execution.
Those cross-builds establish compilation only, not platform-equivalent runtime
evidence. Add native checks only when official runner labels and successful
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
target. The macOS cross-build establishes compilation only, not native runtime
evidence. Add a native check only when an official runner label and successful
native-run evidence are available.

View File

@@ -1,67 +1,54 @@
# Integration: Audita
## Purpose
Define the Audita adapter contract used by the `polish` stage.
Narratio invokes Audita v2's `process INPUT --glossary PATH --output PATH`
subprocess for `polish`. The configured timeout and parent cancellation bound
execution. Narratio passes explicit `--config`, `--overlay`,
`--proposal-profile`, `--validation-profile`, `--transcript-description`,
`--output-schema`, and `--modules` only when selected. An omitted module list
leaves Audita's default sequence in place; `modules: []` passes an explicit
empty argument and runs no correction modules. Order and repetitions are kept.
Narratio passes no debug, work-directory, or reporting flags, leaving those
choices to Audita's own configuration.
## External Boundary
Audita owns its ConfigKit file, backend and profile definitions, native config
discovery, and their validation. Narratio does not create an Audita
configuration or infer downstream overlay/profile names from its own selected
pipeline overlay. An empty Narratio Audita selector adds no CLI override.
The [configuration reference](../config.md#audita-selection-and-credentials)
owns Narratio's fields and defaults. Audita's
[v2 migration guide](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/migrations/v2.md)
owns its downstream file format.
Narratio invokes `audita process` as a subprocess for each polish operation.
The configured timeout and parent cancellation bound the invocation. Internal
runner composition is documented in
[the adapter implementation guide](../internal/adapters.md).
Each name in `pipeline.audita.credentials` is a ConfigKit secret reference
resolved when Narratio loads the pipeline. Narratio forwards only the selected
names and their captured, nonempty values under those exact names, in the
minimal child environment; it never maps them to a fixed alias or rereads the
parent environment. Every name is registered for captured stdout/stderr and
failure-diagnostic redaction. Audita resolves credentials for all configured
backends, including unselected ones, so operators must list every backend
credential that its selected configuration needs. The downstream files and
provider output remain Audita's responsibility, outside Narratio's
captured-stream redaction boundary.
## Request Contract
`PolishRequest` carries:
- required transcript/glossary/output/work-dir paths;
- optional report path (required when report mode is enabled);
- generated config and stdout/stderr log paths;
- optional per-invocation module override.
The corrected transcript is the only canonical polish output. A successful
invocation must produce a valid JSON transcript with a top-level `segments`
array within Narratio's 64 MiB bound. Exit zero is accepted even if Audita's
ordinary reports describe recoverable model errors or no corrections. Narratio
does not parse report status, counts, or Markdown. It discovers only
`errors.yml`, `recoveries.yml`, and `report.md` next to the requested output,
each subject to a 16 MiB read bound. Missing reports are allowed, for example
when Audita reporting is disabled externally. Available reports are diagnostic
references, not canonical artifact outputs. See Audita's
[ordinary output contract](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/integrations/output-files.md)
for their schemas, and [operations](../operations.md) for placement and
publication.
The constructed runner owns static Audita settings: binary, timeout,
credentials, default modules, model and endpoint settings, validation and output
settings, report mode, and concurrency. The `polish` stage supplies only
invocation-specific paths and may override modules for that invocation.
On nonzero exit, Narratio preserves the original subprocess error and
available run-local evidence; it never promotes a transcript merely because
the file exists. The adapter returns safe paths, exit details, and selection
metadata on failure. Narratio's `audita.generated.v2` YAML records the
invocation, including arguments, paths, and credential names but never values;
it is a diagnostic record, not a generated Audita config file.
## Result Contract
`PolishResult` returns:
- processed transcript path;
- optional report path;
- work dir and generated-config/log paths;
- exit code, duration, binary provenance;
- adapter metadata map.
## Validation and Failure Semantics
Construction fails for invalid static config values, including:
- empty binary;
- non-positive timeout;
- invalid base URL;
- invalid output schema;
- invalid work-dir retention value;
- invalid concurrency values.
Run fails for:
- missing required request paths;
- missing required credential env var when configured (`llm_api_key_env`);
- subprocess execution failure;
- invalid processed transcript JSON (`segments` array required);
- invalid report JSON when reporting is enabled.
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
MiB. Both must be regular files without symlinked path components.
Failure results still include output/log/config/exit metadata for diagnostics.
## Deterministic Behavior
- CLI args are built from runner config + request in a fixed order.
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
- Manifest writes are stage-owned; adapter itself is stateless.
## Configuration
Operator-selected values are defined under `pipeline.audita.*` in the
[configuration reference](../config.md#pipeline).
Maintained example with Audita config:
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
Maintained Narratio examples: [full annotated pipeline](../../examples/pipeline.full.annotated.yml)
and [production-shaped pipeline](../../examples/pipeline.production.yml).

View File

@@ -7,14 +7,14 @@ lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
Canonical Notarius v0.6.0 references:
Canonical Notarius v0.7.0 references:
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/cli.md)
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/subprocess.md)
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/dnd-pipeline.md)
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/run-result.md)
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/json-output.md)
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/dnd-spell-catalog-overlays.md)
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
records the exact current constraints for all ten D&D lanes. Treat the linked
@@ -25,19 +25,38 @@ duplicate the complete schemas.
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
configuration path, input path, output directory, and working directory to
absolute paths. Narratio requires the Notarius v0.6.0 CLI contract when
references are configured and invokes each binding as a separate argument
before `--json`:
absolute paths. Configured references require support for the `--reference`
contract introduced in Notarius v0.6.0. Cache controls require support for the
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
cache integration baseline, not an unconditional minimum version; Narratio
does not impose a runtime version gate. Each reference binding is a separate
argument before `--json`. When `pipeline.notarius.config_profile` is present,
its exact value is another argument pair before `--json`:
```text
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--reference <selector>=<verified_snapshot_path>]... --json
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
```
Reference paths are absolute invocation-local snapshots streamed from the
manifest-verified canonical files prepared inside the current Narratio session
workspace. Narratio verifies snapshot checksum and size before and after the
subprocess, and passes only configured bindings, ordered lexically by normalized
selector, as direct argument-vector entries without shell interpretation. A CLI
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
option requires a compatible Notarius executable, but deployments that omit the
field acquire no additional version requirement. Omitted or null configuration
adds no flag. Supplied empty, `default`, and named selectors are forwarded
without normalization so Notarius can apply its own ConfigKit semantics. This
option selects a Notarius configuration overlay, not a PromptKit profile through
`--llm-profile`, and Narratio never substitutes its own selected overlay name.
A missing or invalid downstream overlay is an ordinary Notarius subprocess
failure: Narratio does not retry without the selector or fall back to another
profile.
Reference paths are absolute stable snapshots streamed from manifest-verified
canonical files prepared inside the current Narratio session workspace. They
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
full lowercase content digest. Narratio verifies snapshot checksum and size
before and after the subprocess, reconstructs missing or corrupt regular files
from a still-valid prepared source, and passes only configured bindings,
ordered lexically by normalized selector, as direct argument-vector entries
without shell interpretation. Symlinks, directories, other non-regular
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
binding takes precedence over a matching external path in Notarius
configuration. Narratio never emits `--without-reference`.
@@ -75,9 +94,42 @@ It does not pass a Narratio session ID or run `notarius config validate`
automatically; the configured working directory and Narratio's minimal child
environment apply to the subprocess.
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
Narratio neither reads nor synthesizes its configuration. Use explicit
persistent roots in Notarius when reuse must span subprocess invocations.
Changing a reference path/digest or other upstream identity input correctly
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
replacement for the active source; it does not delete unrelated plans or
checkpoint state.
When `pipeline.notarius.credentials` is absent, null, or empty, Narratio adds no
credentials to that environment. When configured, each list entry names an
environment variable whose already-loaded nonempty value is forwarded under
that exact name. Operators must use the names referenced by the selected
Notarius/PromptKit providers. Narratio does not inspect the Notarius configuration
to infer providers or destination names, and it does not forward unrelated
credentials from its parent environment or secrets directory. A Narratio
overlay replaces the whole credential list; `[]` clears it.
Every selected name is registered with Narratio's shared streaming redactor.
Credential bytes written by the child to captured stdout, captured stderr, or
stderr-derived failure diagnostics are replaced with `<redacted>`, including
custom names not known to the built-in sensitive-name list. Narratio does not
serialize the credential into arguments, stage metadata, manifests, or
generated configuration.
This captured-stream protection is not a filesystem sandbox. Notarius remains
responsible for keeping credentials out of files it writes directly, including
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
arbitrary downstream files and cannot promise redaction of transformed or
encoded credentials.
## Accepted Result
Narratio's supported invocation baseline is Notarius v0.6.0. The accepted
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
receipt remains `notarius.run-result.v2`; reference flags do not change the
receipt or ten-lane output contract. The receipt
must identify the configured pipeline, and its `index_file` must be exactly
@@ -102,6 +154,8 @@ version, and, when configured, module key. Missing, duplicate, rejected, or
incompatible required lanes fail extraction even if Notarius exited zero. A
configured lane whose v2 validation summary is `rejected` or `incomplete` also
fails extraction.
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
preserves that valid summary rather than treating it as a missing field.
Unconfigured lanes may remain in the preserved bundle but do not become
selectable Narratio sources.
@@ -130,3 +184,18 @@ Configuration fields and defaults are in [Configuration](../config.md).
Operator paths, rerun procedures, and bundle retention are in
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
for failure recovery.
## Opt-in Integration Test
The production-boundary cache test requires an absolute path to a locally
built Notarius v0.7.0-compatible executable. It uses only a local
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
```sh
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
GOWORK=off go test ./internal/stage \
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
```
When the variable is unset, the normal test suite skips this test and never
searches `PATH`, a sibling checkout, the network, or a real provider.

View File

@@ -17,11 +17,20 @@ Narratio sends an HTTP `POST` to the configured transcription URL using
The server must return a `2xx` response whose body is valid JSON. Narratio does
not currently require a more specific response schema at this boundary.
The transcription URL must be an absolute `http` or `https` URL. The audio body
is streamed through a fresh multipart writer for every attempt, so its memory
use is bounded by the transport buffer rather than by the complete audio file.
The transcription URL must be an absolute `http` or `https` URL. Each
application attempt creates one bounded multipart framing and streams the audio
from an open file between that framing; it does not buffer the complete audio
or spool a multipart copy. The file field precedes the language field.
WhisperX response acquisition is capped at 10 MiB.
The initial request body and any transport-requested replay use the same
multipart boundary and framing. Every body opens an independent audio handle
and replay starts at the beginning. Narratio closes all bodies created for the
attempt on response, failure, timeout, or cancellation, including when a
transport abandons an earlier body. Genuine source read/close failures remain
request failures; deliberate early closure of an abandoned original does not
poison a valid replay.
## Request And Result Contract
Each adapter request identifies a speaker, a readable audio file, and the
@@ -43,6 +52,11 @@ transcript output.
- Narratio waits the configured retry delay between attempts and aborts that
wait when the parent context is canceled.
Transport replay occurs inside one application attempt and does not consume an
entry from the configured retry budget or increment the reported attempt
count. The standard HTTP redirect policy remains in effect; body replay can
also support body-preserving redirects.
## Validation And Failure Semantics
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
@@ -55,12 +69,19 @@ failure to install the output causes the transcription to fail. Errors include
attempt context, and the result retains attempts, final status when available,
and elapsed duration for diagnostics.
Replay makes a fresh request body available to the HTTP transport; it does not
provide an exactly-once server guarantee and does not prove recovery from every
proxy or protocol-level failure. A server may have received work before the
client learns that a request failed. WhisperX operations should therefore be
safe to retry for the same input when practical.
## Determinism And Concurrency
Each audio request has stable multipart field names, and successful bytes are
installed atomically. The transcribe stage may process speaker files in
parallel, bounded by the configured concurrency. It records results in stable
speaker order after all work completes; any speaker failure fails the stage.
parallel, bounded by the configured concurrency. Each validated speaker is
checkpointed independently; any speaker failure still fails the aggregate
stage, while committed earlier speakers remain reusable on an ordinary retry.
## Related Canonical Docs

View File

@@ -65,6 +65,18 @@ Notarius is composed only when extraction is enabled; the extract stage owns
prepared reference resolution, receipt, bundle, and configured-lane policy.
The adapter validates the ordered selector/absolute-path pairs and is the sole
owner of serializing them as repeated `--reference` arguments before `--json`.
It also serializes an effective resume request as `--resume` and a nonempty
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
adapter, resolves force precedence before constructing the request.
Its request may also carry already-resolved ConfigKit secret references.
Audita's request carries the same kind of list. Both adapters use the shared
subprocess helper to convert their selected references into exact-name child
environment overrides and sensitive-name registration. That helper rejects
unresolved or empty programmatic references, but neither adapter rereads the
parent environment or inspects downstream configuration to discover names.
Empty lists add no credential overrides. See the public [Audita](../integrations/audita.md)
and [Notarius](../integrations/notarius.md) contracts for selection and
downstream-file boundaries.
Application-owned object-store construction loads configured filesystem
secrets before invoking the invocation's storage factory. Production commands
@@ -78,15 +90,15 @@ in-memory backend without changing package state.
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
- Shared subprocess execution keeps private lifecycle mechanics separate from
Narratio's environment, diagnostics, redaction, and error policy. It starts
an owned process group on Linux/macOS or a
kill-on-close job object on Windows. Every terminal path disposes of that
owned tree before returning. After a natural leader exit, Unix checks for
remaining group members and uses bounded graceful then forceful termination;
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
an owned process group on Linux and macOS. Every terminal path disposes of
that owned tree before returning. After a natural leader exit, the runner checks
for remaining group members and uses bounded graceful then forceful termination.
Cancellation, deadlines, and
diagnostic limits use the same terminal disposal path without losing their
original result classification. Child environments contain only the execution
baseline and adapter-specified values; configured credentials are explicit
sensitive values. Stdout and stderr are redacted while streaming into separate
baseline and adapter-specified values; configured credentials are resolved
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
departed leader's descendant. Unsupported platforms reject owned command
execution.

View File

@@ -58,10 +58,12 @@ Configured definitions are always registered. Without an explicit selection,
the effective analyze set contains enabled definitions. With `--artifacts`, the
exact named configured definitions become the effective set for that invocation,
regardless of their `enabled` value. The effective-set resolver itself does not
expand dependencies; the analyze work planner closes those targets over their
configured prerequisite graph. Availability is separate from executability.
Configuration may normalize a family selection into its concrete generated
members before this resolver runs. The effective set retains optional family
expand dependencies. Configuration has already materialized the sorted union of
explicit ordering edges and configured-artifact input edges, and the analyze
work planner closes targets over that one concrete prerequisite graph.
Availability is separate from executability. Configuration may normalize a
family selection into its concrete generated members before this resolver runs.
The effective set retains optional family
and character origin metadata, but its keys, catalog sources, and runtime
lookups remain concrete configured-artifact identities.
Family publish policies are likewise expanded into ordinary configured-source

View File

@@ -8,30 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
`LoadPipeline` assembles and validates a pipeline in this order:
1. Parse the root YAML into a presence-aware composition tree. The tree retains
source names, full field paths, node kinds, declaration order, and explicit
zero, false, empty-map, and empty-list values.
2. Decode the root-only `composition` envelope, validate its declaration shapes
and cross-field profile semantics, then clone the root without that envelope
as the ordinary pipeline body. The parsed source document is not mutated. A
load option retains the difference between omitted and explicitly empty
profile selection.
3. Open each import relative to the root pipeline directory through the
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
extension and cannot traverse, use symlinks, repeat a file, import the root,
or contain another composition envelope.
4. Resolve and structurally parse every declared profile overlay through the
same confined regular-file boundary. Missing or malformed unselected
overlays fail the load. Overlays cannot contain a composition envelope.
5. Additively merge the root body and imports. Distinct map leaves compose;
repeated scalar or list paths and node-kind disagreements are conflicts.
6. Select exactly one declared profile from an explicit option or the default,
then recursively merge its overlay. Overlay leaves replace base leaves,
lists are atomic replacements, and null or kind changes fail.
7. Emit deterministic canonical YAML and strictly decode it into
`PipelineConfig`.
8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from
the root pipeline file, and digest the normalized effective mapping.
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
enabled imports, and the caller's overlay selection. Omitted, empty, and
exact `default` selections use the base; named overlays use the fixed
`overlays/<name>.yml` convention.
2. ConfigKit parses the root and declared imports, additively merges base
sources, applies only the selected overlay, and strictly decodes the result.
3. The definition resolver applies recorded entry-level and inherited defaults,
runs deterministic domain normalization, records derived publish destinations,
and resolves ordinary paths relative to the root pipeline file.
4. The definition validator performs read-only checks that require only the
pipeline file. Checks requiring canonical-party family expansion remain at
the existing explicit pipeline/campaign resolution boundary.
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
declared by multiple base files, adapts source ownership records, attaches
runtime-only state, and computes the normalized effective digest.
This ordering preserves monolithic configuration behavior. Moving a field to
an imported fragment changes its source ownership, not its path base, default,
@@ -53,38 +44,55 @@ download retain that exact pipeline object and its private provenance. Removing
a temporary downloaded session file therefore cannot invalidate the resolved
pipeline or campaign context.
Campaign and session files use independent ConfigKit definitions with imports
and overlays disabled. Campaign files normally load directly from their source;
session files are first read through the 8 MiB bounded-read boundary because the
application must reject unresolved template placeholders before passing an exact
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
the same typed loader and retain the existing post-load identity expectations,
stable-input precedence, and source-relative path semantics.
The application also has a separate read-only inspection resolver for `config
validate`, `config show`, and `config sources`. It uses the same production root/profile and
campaign selection functions, but never routes through session discovery,
remote-session download, secret loading, adapter composition, workspace
validate`, `config show`, `config sources`, and `config artifacts`. It uses the
same production root, overlay, and campaign selection functions, but never routes
through session discovery,
remote-session download, adapter composition, workspace
initialization, manifest access, or cleanup. A pipeline with retained artifact
family declarations must resolve its selected campaign before ordinary pipeline
validation, which expands its canonical-party members and generated publish
rules. A pipeline without those declarations may be validated by itself.
Secret-directory bootstrap remains invocation-scoped and happens before the
ordinary ConfigKit load because explicit typed references must resolve even for
inspection.
`MarshalEffectivePipeline` is the configuration-owned projection for `config
show`. It serializes the typed, defaulted effective mapping through the
deterministic composition renderer, then removes resolution-only artifact
family declarations. The result contains no composition envelope or private
provenance fields and has one trailing newline; commands do not marshal runtime
objects directly.
show`. It serializes one explicit typed safe projection with the existing YAML
encoder. The projection has no slots for resolution-only artifact family
declarations or private provenance. The result has one trailing newline;
commands do not marshal runtime objects directly.
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
safe provenance projection for `config sources`. Pipeline ownership begins with
the complete logical field paths retained during composition and classifies
each contributor as root, import, profile, or centralized default. The
the complete logical field paths projected from the schema and classifies
each contributor as root, import, overlay, or centralized default. The
projection replaces generated concrete member paths with paired family and
canonical-party records, and does the same for generated publish rules.
Campaign records identify campaign-owned fields and party inputs; canonical
derived players point to the party source. The application command only joins these sorted
records with selection metadata and never reparses configuration files.
`config diff` uses a paired profile loader that parses the root, imports, and
declared overlays once, then clones the additive base before independently
selecting, decoding, defaulting, and finalizing each profile. When campaign
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
`config artifacts`. It derives source identity from each map key, carries sorted
effective dependencies, aggregates ConfigKit definition contributors, and adds
family/character/party provenance for generated members. It does not inspect
files named after artifacts or infer another runtime graph.
`config diff` uses a thin paired loader that performs two independent ConfigKit
loads. When campaign
resolution is needed, the command loads one selected campaign and party and
expands both effective pipelines from that same party value. The configuration
owner projects each normalized effective mapping into sorted logical paths;
owner projects each normalized effective mapping directly from that same typed
projection into sorted logical paths using JSON number preservation;
mapping leaves are compared individually while sequence values remain atomic.
Values are compact deterministic JSON representations for command output, not
raw YAML fragments, ownership records, or secret material. A differing digest
@@ -98,13 +106,16 @@ configured roster authority; player and session-party fields are unsupported.
## Canonical Party Domain
`ParseParty` is the package-owned boundary for parsing a party source. It
requires a top-level `schema_version` and strictly validates the
`narratio.party.v1` contract into ordered character domain values. The
canonical value retains a separate exact byte copy of its source so consumers
can materialize the authored party document without reserializing it. Its
`PlayersYAML` method deterministically derives the versioned players-only
projection.
`ParseParty` is the package-owned boundary for parsing a party source. Its
single-source ConfigKit definition disables imports and overlays, requires
`schema_version` through `VersionRule`, and strictly decodes the
character-keyed typed schema before applying Narratio's party-domain checks.
Successful metadata coordinates recover authored character-map order without a
second YAML parse; class and alias slice order comes directly from typed decode.
The canonical value retains a separate exact byte copy of its source so
consumers can materialize the authored party document without reserializing it.
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
players-only projection.
An unversioned source is rejected before resolution, family expansion, or
prepare-stage materialization.
@@ -117,58 +128,67 @@ for core pipeline/session rules, external adapters, publish/storage, Notarius,
Scriptorium, and shared scalar/path checks. These are package-local ownership
boundaries rather than extension points or a general validation framework.
Validation fails fast and does not define which of several independent invalid
fields is reported first. Map-backed declarations are nevertheless traversed
deterministically. Notarius validates scalar policy, builds normalized reference
bindings and output lanes independently, and installs all candidate values only
after every check succeeds. Scriptorium validates artifact declarations and
outputs, dependencies and inputs, variables, referenced outputs, and the final
Validation is read-only, fails fast, and does not define which of several
independent invalid fields is reported first. Map-backed declarations are
nevertheless traversed deterministically. Reusable domain normalization is an
explicit preceding operation for both loaded and programmatically constructed
pipelines. Notarius normalization commits canonical scalar, reference, and output
values only after the complete normalization succeeds; its validator then checks
that candidate without changing it. Scriptorium validates artifact declarations
and outputs, dependencies and inputs, variables, referenced outputs, and the final
dependency graph in deterministic artifact order.
Diagnostics identify the relevant semantic field. Composition conflicts also
Diagnostics identify the relevant semantic field. Merge conflicts also
retain every claiming source needed to correct an additive ownership conflict.
Complete prose, punctuation, formatting, and private validation choreography are
not compatibility contracts.
## Diagnostics And Runtime Metadata
Syntax, duplicate-key, composition, conflict, and schema failures include the
Syntax, duplicate-key, merge, overlay, and schema failures include the
relevant source name and full field path. Additive conflicts report every
claiming source so operators can repair the split without repeatedly
rediscovering additional conflicts.
The loaded pipeline retains private runtime metadata for the absolute root
path, ordered imports, selected profile name and selection source, selected
overlay, contributing sources, effective digest, and leaf ownership. Base
path, ordered imports, selected overlay name, source, and path, original
ConfigKit metadata and source snapshots, effective digest,
and leaf ownership. Base
leaves retain their root/import owners, replaced leaves belong to the selected
overlay, and centrally supplied values use the synthetic `default` owner. This
metadata does not participate in YAML decoding or alter the public
configuration model.
The effective digest is SHA-256 over deterministic canonical YAML produced from
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
execution, but the digest substitutes their normalized logical values captured
before root-relative resolution, so relocating an equivalent configuration
bundle does not change provenance. Because composition and resolution metadata
are private, the digest excludes source layout, profile name, and ownership.
Configuration stores environment variable names rather than resolving raw
credentials, so raw secret values are neither loaded nor hashed.
bundle does not change provenance. Because ConfigKit and resolution metadata
are private, the digest excludes source layout, overlay name, and ownership.
Configuration stores credentials as ConfigKit typed references. A successful
load resolves their values inside the opaque typed boundary, while this safe
projection serializes only `EnvName()` identities. Credential-value changes are
therefore never hashed; reference-name changes remain ordinary provenance.
The optional Notarius credential follows the same path even for disabled or
resumable extraction; normalization copies the opaque reference without reading
or resolving it again. Direct YAML serialization has a configuration-owned safe
projection so only the reference name can leave this boundary.
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
for later runtime expansion.
## Test Surfaces
`composition_test.go` protects the presence and merge algebra independently of
the public schema. `pipeline_composition_test.go` exercises explicit imports,
confinement, conflicts, strict decoding, metadata, and root-relative path
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
all-overlay validation, overlay behavior, provenance, option propagation, and
effective-digest stability. Application configuration-loader tests protect the
`pipeline_configkit_test.go` exercises real imports and fixed-path overlays,
base selection, unselected-overlay isolation, source attribution, and Narratio's
unique artifact/family base-declaration rule. Existing focused configuration
tests protect defaults, explicit-value semantics, strict decoding, validation,
metadata, and root-relative path behavior. Application configuration-loader tests protect the
single-read boundary by changing the pipeline file after its initial load and
confirming local session resolution retains the original pipeline. Other
configuration tests continue to protect defaults and validation after assembly.
`party_test.go` protects the versioned party schema, domain invariants, and
deterministic players projection without involving campaign or runtime wiring.
`party_test.go` protects the versioned typed party schema, domain invariants,
authored character order, retained bytes, and deterministic players projection
without involving campaign or runtime wiring.
`party_resolution_test.go` protects campaign-owned party loading, canonical
input restrictions, source provenance, and virtual players input selection.

View File

@@ -23,9 +23,11 @@ portable opaque segments. Unsafe legacy identities are rejected with migration
guidance rather than being normalized into a different workspace or remote
namespace.
Decoding is strict at every typed manifest field. Unknown fields are rejected;
the schema version remains `narratio.session-manifest.v1` and there is no
compatibility reader that silently drops retired state. Cleanup progress uses
Decoding rejects unknown typed fields, including informational fields inside
`effective_config` and its nested selection. The schema version remains
`narratio.session-manifest.v1`; there is no compatibility reader that silently
drops retired fields. See [overlay migration](../operations.md#migrating-manifest-overlay-provenance)
for deliberate migration of retired selection provenance. Cleanup progress uses
only the cleanup entry in `stages`.
Every persisted session manifest has a non-nil `stages` map. Each entry is
@@ -50,6 +52,45 @@ The model admits these stage states:
- `stale`
- `interrupted`
### Transcribe-owned speaker state
A `transcribe` stage record may carry `transcribe_state_version: 1` and a
`transcribe_speakers` map keyed by the existing derived speaker ID. The version
is the authority marker: a versioned empty map means that no earlier speaker
completion is reusable. Absence of both fields is accepted as legacy
aggregate-only state so current Narratio can read older manifests.
Each map entry represents one completed speaker only. It binds a versioned
fingerprint to the session-relative audio path, audio SHA-256 and size, one
canonical `transcript_raw` output with its path, SHA-256 and size, and the
producer run ID. Paths must be canonical and session-relative, checksums must
be lowercase SHA-256 values, and the record and output producer identities
must agree. Other stages cannot carry this transcribe-owned state.
Audio size is nonnegative; WhisperX determines whether that audio can be
transcribed. Output size remains positive and the transcript must be valid JSON.
Transcribe reconciles this collection against the current prepared audio set
and observable WhisperX semantics. Missing or corrupt regular outputs become
work; unsafe or non-regular evidence is rejected. It saves the reconciled
collection before HTTP dispatch and saves a detached candidate after each
validated completion. Only after a save succeeds is the candidate installed as
live authority. Canonical bytes produced before an unsuccessful save are not a
completion record.
Aggregate transcribe status still describes the whole invocation. A failed,
running, stale, or interrupted aggregate may therefore retain reusable speaker
records, while aggregate outputs are declared only after the current
collection is complete. Ordinary retry reconciles and reuses those records.
Force first checkpoints an empty current collection, preventing later retries
from reviving pre-force results.
Older aggregate-only transcribe success is conservatively regenerated once
when transcribe is selected because it does not bind individual outputs to the
audio bytes used. The session-manifest schema identifier remains
`narratio.session-manifest.v1`; this additive state is accepted by current
Narratio, but older strict-decoding binaries may reject manifests after these
fields have been written.
### Analyze-owned artifact state
Every `analyze` stage record carries `analyze_state_version: 1` and an
@@ -154,12 +195,31 @@ establish a current immutable snapshot.
- writes through a sibling temporary file and atomic replacement visibility;
- updates `updated_at` on save.
### Checkpoint persistence boundary
Analyze and transcribe share one private checkpoint operation in
`internal/stage`. Their typed callers still decide when to checkpoint, validate
their collections, populate stage-owned fields, and serialize access. The
shared operation copies the manifest and its checkpoint-relevant mutable
collections, applies the typed update to the copied target record, and calls
the existing `manifest.Store` exactly once. Only after that save succeeds does
it promote the saved target record and manifest `updated_at` into live state;
it preserves an existing target record's identity and leaves unrelated live
stage records in place.
The copy boundary includes stage records, input and artifact slices, aggregate
output/log/configuration slices, and the typed analyze and transcribe
collections that callbacks or `LocalStore.Save` may modify. Metadata and other
pointer fields remain shared read-only because neither checkpoint callback nor
the store mutates them. This is a bounded execution helper, not a general
manifest clone or a transaction across transcript/artifact files.
Manifest replacement provides atomic visibility, not a crash- or power-loss
durability guarantee. Callers reload after a returned save error before retrying.
An analyze checkpoint error leaves the prior manifest as per-artifact authority.
The runner then attempts to persist the bounded failed/stale retry collection;
if that terminal save also fails, its error is joined with the originating
failure and the next invocation must reconcile from the last readable manifest.
A transcribe or analyze checkpoint error leaves the prior manifest collection
as authority. The runner then attempts to persist the aggregate failure; if
that terminal save also fails, its error is joined with the originating failure
and the next invocation must reconcile from the last readable manifest.
## Execution Semantics
@@ -193,11 +253,11 @@ only fields returned by the new result. Marking a record stale does not clear
those details because resume validation and diagnosis may still require them
before execution begins.
Aggregate lifecycle clearing deliberately preserves the analyze-owned
per-artifact collection. This lets later reconciliation replace only evaluated
entries without erasing unrelated current results. Other stages retain their
existing aggregate-only lifecycle behavior and are forbidden from carrying the
analyze-specific fields.
Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
speaker collection and the analyze-owned per-artifact collection. This lets
later reconciliation replace only non-current entries without erasing durable
completions. Other stages retain aggregate-only lifecycle behavior and are
forbidden from carrying either collection.
A stage may explicitly return a skipped disposition and stable reason. The
runner persists that outcome in the session manifest, clears older outputs,
@@ -228,7 +288,7 @@ evidence and focused durable validators:
| Work | Reuse authority | Focused owners |
| --- | --- | --- |
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
| transcribe | aggregate semantic fingerprint | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
| transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
@@ -248,10 +308,10 @@ after such a private content change.
The session manifest is the authoritative stage-progress ledger across
invocations. It retains the most recently resolved invocation's bounded
configuration provenance. It identifies the selected profile name and source
(`default` or `cli`) plus the effective configuration digest, but never a raw
secret or profile content. This provenance is informational: it does not
participate in stage resume or cache decisions. A profile change therefore
configuration provenance. It identifies the selected overlay name and source
(`explicit` when selected) plus the effective configuration digest, but never a raw
secret or overlay content. This provenance is informational: it does not
participate in stage resume or cache decisions. An overlay change therefore
invalidates only stages whose semantic configuration changed. When a private
external-tool model, module, prompt, or profile changes behind an unchanged
configured identifier, use `--force` for the affected work.
@@ -310,6 +370,7 @@ where the last session transition can require operator interpretation.
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
`internal/artifacts/current_state_commit.go`,
- Persistence and validation: `internal/manifest/store.go`
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
- Package tests: `internal/manifest/*_test.go`
- Assembled execution behavior: `internal/app/runner_test.go`,
`internal/app/run_stage_test.go`

View File

@@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting.
| --- | --- | --- |
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and overlays, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
@@ -53,11 +53,12 @@ adapters. The normative rules for these relationships remain in
[Architecture](../policy/architecture.md).
Command execution normalizes one private dependency value per invocation.
Configuration reads, secret-directory reads, remote downloads, object-store
construction, and restore phase functions use that value throughout the
command. Public command entry points supply production defaults, while focused
tests replace only the external or sequencing boundary they exercise without
mutating package state.
Pipeline configuration-path discovery, configuration reads, secret-directory
reads, remote downloads, object-store construction, and restore phase functions
use that value throughout the command. Public command entry points supply the
canonical discovery and production defaults, while focused tests may provide
invocation-owned discovery candidates or replace only the external or sequencing
boundary they exercise without mutating package state.
Pipeline execution and `session plan` share the same inclusive contiguous-range
model and the same pure pre-execution decision over recorded status, force, and
@@ -105,8 +106,8 @@ so any earlier replacement can stale a prior cleanup result.
## Focused Documentation
- [Configuration Internals](configuration.md): pipeline composition, import
confinement, field ownership, decoding, and root-relative path semantics.
- [Configuration Internals](configuration.md): pipeline imports and overlays,
field ownership, decoding, and root-relative path semantics.
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
failure behavior, and test surfaces.
- [Artifact Internals](artifacts.md): source identities, runtime catalog,

View File

@@ -10,7 +10,9 @@ dependency order, and safely materialize validated outputs.
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
party artifact families have already expanded into this map during
configuration resolution, including corresponding member dependencies and
rewritten member-artifact input sources
rewritten member-artifact input sources. Each artifact's dependency list is
already the normalized union of explicit ordering edges and configured-artifact
input edges.
- optional selected artifact keys supplied through the stage environment
- built-in, configured, extraction, and previous-session source references in
artifact inputs
@@ -51,6 +53,9 @@ Supported source families:
manifest record and durable output pass the configured-artifact evidence
contract; an incidental or stale canonical file is unavailable.
- validates selected artifact dependency order (cycle-safe topo ordering).
- consumes only the concrete dependency graph finalized by configuration;
planning, prerequisite closure, execution, reuse, invalidation, and semantic
evidence do not infer a second graph from inputs.
- resolves required/optional inputs per artifact source definition into an
ordered semantic identity. Each identity records the configured input name,
canonical source ID, required policy, explicit presence, source contract,
@@ -110,7 +115,11 @@ Supported source families:
- before the first adapter call, checkpoints the reconciled starting collection
when it removed configuration records or cleared obsolete authority. After
each successful artifact, checkpoints the complete collection before making
that artifact available to the next scheduled dependent.
that artifact available to the next scheduled dependent. Analyze supplies
its typed update to the shared
[checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary)
and refreshes its working and committed collections only after the save
succeeds.
- keeps adapter output in the invocation's run-local analyze directory until
it is a safe, non-empty, bounded regular file with a calculated checksum and
complete output contract. Canonical replacement uses the shared atomic file

View File

@@ -19,12 +19,17 @@ procedures belong in [Operations](../operations.md).
1. resolves the final trimmed transcript from the shared artifact catalog;
2. resolves every configured prepared reference through the shared
manifest-authoritative identity resolver before creating run-local output;
3. streams each verified reference into an invocation-local snapshot and
rejects any source change observed while copying;
3. resolves each reference to a stable content-addressed session snapshot,
reusing verified bytes or reconstructing a missing/corrupt regular entry,
and rejects any source change observed while copying;
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
including sorted reference identities;
including sorted reference identities and the explicit downstream
configuration-profile selection;
5. creates a run-local staging directory and invokes the injected
`notarius.Runner`;
`notarius.Runner`, carrying the optional already-resolved credential
reference without exposing its value, the optional exact configuration
profile, plus effective resume and chunk-cache controls after force
precedence;
6. revalidates the reference snapshots, then validates the v2 successful
receipt, confined index, management documents, configured required lane
descriptors, validation summaries, and regular payload files;
@@ -41,14 +46,22 @@ Narratio run ID, the resolved trimmed-input identity, and invocation
fingerprint. The input identity binds the exact transcript bytes, canonical
source ID, producer stage/output/run identity, and resolution provenance.
Reference metadata contains only selector, source ID, canonical session-relative
path, checksum, and size; adapter requests receive selector and absolute
invocation-local snapshot path, never payload contents. Snapshot bytes must
path, checksum, and size; adapter requests receive selector and absolute stable
snapshot path, never payload contents. Snapshot bytes must
match the prepared identity both before and after Notarius runs, so a concurrent
prepared-file replacement cannot make recorded provenance describe different
bytes from those supplied to Notarius.
Validation completes before the single durable rename, so a rejected result
cannot expose a partial durable bundle.
Snapshot destinations are owned by `internal/artifacts` under
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
or timestamp and can be reused across invocations. Identical bytes and basename
share a verified file; changed bytes select a different digest directory.
Unsafe path components and non-regular conflicts fail without recursive
deletion. Snapshot paths are delivery details and never become manifest output
evidence or prerequisites for reuse of an already complete extract bundle.
Any executed extraction outcome that replaces a different effective outcome
marks succeeded analysis and delivery dependents stale. Render is an independent
sibling and remains current. Repeating the same disabled self-skip with no
@@ -59,17 +72,28 @@ outputs is stable and does not repeatedly invalidate dependent stages.
Before the focused validator runs, the application compares extract's versioned
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
reference selector/source mappings, sorted declared output contracts, and each
canonical `narratio.extraction.<key>` output identity. It excludes executable,
canonical `narratio.extraction.<key>` output identity. It also covers the
explicit Notarius configuration-profile selection; omitted, empty, and exact
`default` selectors share the base-selection identity. It excludes executable,
timeout, working directory, config path, and private Notarius config contents.
Credential presence, reference name, and resolved value are likewise
operational-only and excluded. Ordinary command loading still resolves a
selected reference before resume evaluation, but a credential-only change does
not rerun a valid completed extraction or stale its dependents.
`internal/stage/extract_resume.go` then permits a skip only when the existing
stage record still matches the current byte- and provenance-bearing invocation
evidence. That evidence covers the current direct trimmed-transcript identity,
sorted prepared-reference identities, pipeline identity, and configured output
contracts. The same reference helper and transcript identity are resolved again
for artifact evidence, so changing current transcript bytes, reference bytes,
or producer identity makes the prior extraction obsolete. Operational runner
settings do not invalidate otherwise current durable evidence.
contracts, plus the same downstream profile-selection identity. The same
reference helper and transcript identity are resolved again for artifact
evidence, so changing current transcript bytes, reference bytes, producer
identity, or a named downstream selection makes the prior extraction obsolete.
Operational runner settings do not invalidate otherwise current durable
evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
make a complete extraction non-resumable. If extraction executes, ordinary
runs forward the configured values, while force disables resume and selects
`refresh`.
A valid prepared-reference change makes extraction non-resumable. Missing,
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
@@ -82,10 +106,12 @@ contracts and provenance, regular-file status, and stored checksums. Missing or
obsolete results are non-resumable and run again; unsafe filesystem conditions
return an error rather than silently accepting or replacing data.
Neither contract can observe files imported by Notarius configuration, profile
contents, prompt/module definitions, or other transitive inputs. Operators must
force extraction after changing any such private input behind a stable
identifier.
This sensitivity is limited to the explicit downstream selector. Narratio's
own profile label and whole effective-configuration digest are not blanket
extraction invalidators. Neither contract can observe files imported by
Notarius configuration, profile contents, prompt/module definitions, or other
transitive inputs. Operators must force extraction after changing any such
private input behind a stable identifier.
## Failure Behavior
@@ -110,9 +136,13 @@ recovery.
`internal/stage/extract_resume.go`,
`internal/stage/extract_test.go`,
`internal/stage/semantic_contracts_delivery.go`
- Stable snapshot materialization and verification:
`internal/stage/extract_references.go`
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
`internal/adapters/notarius/subprocess_test.go`
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
`internal/artifacts/extraction_catalog_test.go`
- Composition and downstream behavior: `internal/app/runner_test.go`,
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
- Opt-in Notarius v0.7.0 production-boundary coverage:
`internal/stage/notarius_cache_integration_test.go`

View File

@@ -17,7 +17,11 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Key Behavior
- discovers and validates raw transcript inputs.
- discovers and validates raw transcript inputs. For versioned transcribe
state, discovery requires aggregate transcribe success and consumes only the
aggregate outputs that exactly match the current per-speaker collection.
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
workflows without versioned speaker state retain directory discovery.
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
- merges normalized inputs (`seriatim.Run`) into base transcript.
- validates merged transcript and optional report JSON.
@@ -26,6 +30,8 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Invariants
- merge always consumes normalized forms of raw inputs.
- a versioned transcribe collection is usable only through its complete,
successful aggregate output declaration.
- base transcript must validate before stage success.
- report output is config-gated.

View File

@@ -12,36 +12,41 @@ Run Audita polishing on base transcript and produce polished transcript.
## Outputs
- `transcripts/polished.json`
- optional `artifacts/audita.report.json`
The transcript is the only canonical output. Available `errors.yml`,
`recoveries.yml`, and `report.md` remain beside the run-local transcript and
are recorded as diagnostics, not artifact outputs.
## Key Behavior
- resolves base transcript only from verified merge manifest output evidence.
- invokes an Audita runner configured with static model/runtime options; the
invocation supplies paths and modules.
- invokes an Audita v2 runner with external configuration selectors and
explicitly forwarded provider credentials; the invocation supplies paths.
- validates processed transcript structure (`segments` array required).
- validates optional report JSON.
- materializes canonical outputs; records logs/generated config and adapter metadata.
- validates reported diagnostic paths against the three fixed adjacent names.
- materializes only the canonical transcript; records available reports with
stdout/stderr logs, generated invocation, and safe adapter metadata.
## Invariants
- polished transcript schema validation is mandatory.
- report output is config-gated.
- reports are optional diagnostics; Audita's external configuration may disable
them. A failed invocation never promotes an incidental transcript.
## Resume Semantics
The versioned semantic fingerprint covers the Audita service endpoint, model,
validation model, module set, transcript description, output schema, selected
external configuration path, and canonical polished-transcript identity. Module
ordering is normalized because the configured modules form a set. Audita's
executable, timeouts, concurrency, report and debug behavior, work retention,
and credential environment name are operational and do not invalidate a
successful result.
Version 2 semantic evidence covers Audita's config path, downstream overlay,
proposal and validation profile selectors, ordered module selection (including
the distinction between omission and an explicit empty list), transcript
description, output schema, and canonical polished-transcript identity.
Credentials, executable path, timeout, and diagnostics are operational and do
not invalidate a successful result. Public credential reference names may
participate in configuration provenance but never expose resolved values.
Narratio can fingerprint a selected model, module, or configuration identifier,
but it cannot inspect content that Audita privately resolves behind that stable
identifier. Force `polish` after changing such private content without changing
its identifier.
Narratio fingerprints selections, not the contents of Audita-owned config or
profiles. Force `polish` after changing those files behind unchanged selections.
When polish is selected, older semantic evidence is reconsidered and the stage
reruns; a downstream-only invocation does not eagerly run polish.
## Related Contracts And Tests

View File

@@ -14,12 +14,19 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
## Key Behavior
- discovers prepared audio from manifest inputs or canonical audio directory.
- derives the transcript identity from the prepared `.flac` filename.
- dispatches one WhisperX request per prepared file through a bounded errgroup.
- validates each output as JSON.
- writes run-local outputs then materializes canonical transcript outputs only
after every planned request succeeds.
- discovers prepared audio from manifest inputs or the canonical audio
directory and derives each speaker identity from its `.flac` filename.
- hashes the current audio and reconciles it with the versioned per-speaker
completion collection before starting HTTP work.
- reuses only records whose audio identity, observable WhisperX semantics,
canonical output location, JSON, checksum, and size remain current.
- durably checkpoints the reconciled collection before dispatch, then sends
only missing or obsolete speakers through the bounded worker group.
- validates each successful run-local response, rechecks the audio bytes,
materializes the canonical raw transcript, and checkpoints that speaker
before accepting another completion.
- rebuilds aggregate stage outputs in stable speaker order from the complete
current collection, including records produced by earlier invocations.
## Invariants
@@ -28,21 +35,49 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
- output path returned by adapter must match requested output path.
- an empty adapter result path means the requested path; adapters cannot select
an alternate destination.
- each successful output is validated before stage success, and cancellation or
incomplete dispatch cannot be reported as a successful result.
- candidate collection construction, the shared
[checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary),
and installation into live state are serialized by transcribe's mutex.
Expensive hashing and HTTP work remain outside that critical section.
- a canonical file has no completion authority until its manifest checkpoint
succeeds. A save failure preserves the last successfully committed
collection for ordinary retry.
- each successful output is validated before its checkpoint, and cancellation,
incomplete dispatch, or a failed aggregate save cannot be reported as
aggregate stage success.
- removed audio inputs are omitted from the current collection and aggregate
output list; reconciliation does not delete arbitrary transcript files.
- merge cannot consume a partial versioned collection: it requires transcribe
aggregate success and an exact match between current speaker records and
declared aggregate outputs.
## Resume Evidence
Transcribe records a versioned semantic-configuration fingerprint containing
the Narratio-visible WhisperX service URL and recognition language. Changes to
either rerun transcription and stale its fixed descendants while leaving
prepare reusable. Retry count/delay, concurrency, timeout, credentials, and
diagnostic locations are operational and do not change this evidence.
Each completed speaker record binds the speaker ID, session-relative audio
path, streamed audio checksum and size, the Narratio-visible WhisperX service
URL and recognition language, and the canonical transcript path, checksum,
size, and producer run. Changed audio bytes select that speaker for replacement;
a changed URL or language selects every speaker. Either case stales fixed
descendants through the normal runner decision. Retry count/delay, concurrency,
timeout, credentials, and diagnostic locations are operational and do not
change this evidence.
An ordinary retry reuses every valid speaker checkpoint even when aggregate
transcribe status is failed, running, or stale. A succeeded stage is skipped only
when its stored speaker set and aggregate output evidence also match the current
collection. Otherwise, execution repairs those records without retranscribing
reusable speakers and stales dependents through the normal runner path.
`--force` first durably replaces
the collection with an empty versioned collection, so a later ordinary retry
cannot resurrect pre-force results; speakers checkpointed by that forced
attempt remain reusable.
WhisperX models or private service configuration not exposed by Narratio's
adapter contract cannot be fingerprinted; use `--force` after changing them.
An existing successful transcribe record without evidence reruns once when
selected.
An older aggregate-only transcribe record has no trustworthy audio-to-output
binding. When transcribe is next selected, every current speaker is therefore
generated once and recorded in the versioned collection; later invocations can
reuse it normally.
## Related Contracts And Tests
@@ -51,6 +86,9 @@ selected.
- [Configuration](../config.md#pipeline) owns concurrency and other
operator-selected values.
- Implementation and tests: `internal/stage/transcribe.go`,
`internal/stage/transcribe_reconciliation.go`,
`internal/stage/transcribe_execution_state.go`,
`internal/stage/transcribe_resume.go`,
`internal/stage/transcribe_test.go`,
`internal/stage/semantic_contracts_initial.go`, and
`internal/stage/semantic_contracts_initial_test.go`
`internal/stage/transcribe_reconciliation_test.go`, and
`internal/stage/transcribe_checkpoint_test.go`

View File

@@ -37,6 +37,12 @@ validates and renames the external bundle to the unique immutable Notarius
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
[Operations](../operations.md#extraction-workflow).
Prepared references delivered to Notarius use session-durable paths owned by
`internal/artifacts` beneath
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
content-addressed snapshots are outside run-local layout, survive ordinary
published-run cleanup, and are excluded from publish and restore enumeration.
## Locking
`artifacts.LocalStore` enforces the single-writer session lock via an

View File

@@ -70,18 +70,21 @@ narratio run 2026-04-04
narratio session status 2026-04-04
```
Run, plan, and status output identify the resolved pipeline profile (or `none`)
Run, plan, and status output identify the resolved pipeline overlay (or `none`)
and effective configuration digest. Status distinguishes the current resolved
value from the last value persisted in the session manifest, which helps
diagnose profile switches without changing resume authority.
diagnose overlay switches without changing resume authority.
Before switching an operational profile, compare its effective meaning with the
current selection through `narratio config diff <left-profile> <right-profile>`.
Before switching an operational overlay, compare its effective meaning with the
production base through `narratio config diff default <overlay>`.
The command is read-only and succeeds whether it finds differences or not. Its
sorted records describe defaulted, expanded concrete configuration—not source
file layout—so it can be used to review model, artifact, and publish changes
without creating a session or run. Select the same campaign explicitly when
profiles could resolve different campaign paths; see the [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) for syntax and record format.
without creating a session or run. Use `narratio config artifacts` to review the
concrete artifact identities, definitions, outputs, and dependencies. Select the
same campaign explicitly when overlays could resolve different campaign paths;
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
for syntax and record format.
## Stage Execution and Continuation Behavior
@@ -113,18 +116,19 @@ Execution rules:
perpetually rerun dependent work.
Every aggregate stage except analyze currently provides semantic-configuration
evidence; analyze retains its more precise per-artifact fingerprints and
validator.
evidence; transcribe additionally reconciles per-speaker completion evidence,
while analyze retains its more precise per-artifact fingerprints and validator.
Prepare additionally validates current stable/local/S3 source identity and the
checksums of its durable prepared copies before reuse. Changed bytes, audio
membership, S3 object identity, or missing/tampered copies rerun prepare and
its fixed descendants without requiring `--force`. Changing prepare selection
semantics likewise reruns all fixed descendants; changing
WhisperX language/service identity reuses prepare; changing a Seriatim merge
transformation reuses prepare and transcribe; and changing an Audita model
reuses prepare, transcribe, and merge while rebuilding transcript refinement.
A trim change invalidates both render and extract through the fixed dependency
relation, while a render-only change preserves the extract sibling.
transformation reuses prepare and transcribe; and changing a selected Audita
overlay, profile, or module sequence reuses prepare, transcribe, and merge while
rebuilding transcript refinement. A trim change invalidates both render and
extract through the fixed dependency relation, while a render-only change
preserves the extract sibling.
Operational timeouts, retry/concurrency tuning, executable paths,
workspace/cache/spool placement, reports, diagnostics, and secret values are
@@ -184,7 +188,68 @@ shared session lifecycle. In particular, render does not require Notarius or
Scriptorium, extract does not require Scriptorium, and analyze does not require
the transcription, Seriatim, Audita, or Notarius adapters.
For the common post-transcript development loop, use:
### Transcription continuation and replacement
Transcribe checkpoints each validated speaker independently. If a later
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
save fails, the aggregate stage is not successful and downstream merge cannot
consume the partial set. The last successfully saved speaker records and their
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
After correcting the underlying problem, retry normally:
```bash
narratio run-stage transcribe 2026-04-04
```
Narratio hashes the current prepared audio, validates recorded transcript JSON,
checksum, size, and location, and calls WhisperX only for missing or obsolete
speakers. A changed audio file replaces only its speaker; a changed WhisperX
URL or language makes every current speaker non-reusable. Retry, timeout, and
concurrency tuning do not invalidate completed speakers.
Use force only when every current transcript must be replaced, including after
a WhisperX model or private server-side configuration changes behind the same
URL and language:
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Force durably clears the reusable collection before sending requests. If that
forced attempt fails, a later ordinary retry reuses only replacement speakers
checkpointed after the reset; it does not revive pre-force results. Do not edit
speaker checksums or copy older JSON into the canonical directory to influence
reuse.
The first current Narratio invocation that selects transcribe for an older
aggregate-only manifest retranscribes every current speaker once because the
old record cannot prove which audio bytes produced each file. Subsequent runs
reuse the new versioned evidence normally. The overall session-manifest schema
identifier is unchanged, but older strict-decoding Narratio binaries may reject
a manifest after current Narratio writes the additive speaker fields; retain a
backup or complete the upgrade before relying on downgrade.
Restoring a manifest does not implicitly publish or restore every raw speaker
file. If a recorded canonical transcript is absent locally, ordinary
reconciliation schedules that speaker again. Removing run directories does not
remove canonical checkpointed transcripts and does not by itself invalidate
them.
## Artifact Development Workflows
Choose the narrowest workflow that matches the intended replacement:
| Intent | Command | Preserved and rebuilt state |
| --- | --- | --- |
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
All three stop before publication. Inspect the results and run
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
For the extraction-plus-analysis development loop, use:
```bash
narratio regenerate-artifacts 2026-04-04
@@ -217,6 +282,30 @@ earlier checkpoints are reused and execution resumes with failed, stale, or
missing work in dependency order. `--force` is unnecessary unless the operator
intends to replace otherwise-current artifacts.
### Developing With A Notarius Configuration Profile
A Narratio development overlay can explicitly select a Notarius overlay without
requiring the names to match. For example, `overlays/testing.yml` may contain:
```yaml
notarius:
config_profile: downstream-economy
```
Selecting Narratio's `testing` overlay then supplies `downstream-economy` only
when extraction executes. Switching that explicit selector makes a recorded
extraction non-current and reruns it when `extract` is in the selected stage
range; the usual dependent invalidation then applies. Selecting an overlay does
not create a separate workspace or retain a separate current extraction result.
Narratio cannot observe edits inside Notarius configuration behind an unchanged
selector. Force extraction after such an edit when a replacement is intended.
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
use `analyze` when the existing extraction bundle should be preserved. Field
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
and the exact downstream invocation belongs in the
[Notarius integration contract](./integrations/notarius.md#invocation).
## Artifact Selection
For a configured artifact family, selecting its family key expands to every
@@ -257,9 +346,14 @@ are not selectable or published implicitly.
Configured Notarius references resolve only from the current manifest-backed
prepared inputs. Their canonical locations are `inputs/party.yml`,
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified copies
under `runs/<run_id>/extract/references/` so a concurrent refresh of canonical
prepared files cannot change the bytes consumed by an in-flight invocation.
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
content-addressed copies under
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
SHA-256 digest identifies the reference bytes, so retries and later invocations
reuse the same path while changed bytes select a new path. Older snapshots are
retained and are not current artifact evidence. A concurrent refresh of
canonical prepared files therefore cannot change the bytes consumed by an
in-flight invocation.
For a canonical party, preparation retains the validated authored party bytes
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
The manifest records their checksums separately, with the players input marked
@@ -294,6 +388,104 @@ Notarius extraction validates its run-local bundle and moves it once to the
unique durable bundle path. A pre-existing durable destination is a collision;
no duplicate promotion tree is created.
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
inside an executed Notarius invocation. Ordinary extraction forwards their
configured values. Forced extraction always disables checkpoint loading and
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
Refresh regenerates and replaces only the active input's accepted chunk plan;
it is not cache deletion.
Notarius checkpoint recording must be enabled in the deployed Notarius
configuration. Narratio neither creates a Notarius pipeline/configuration nor
reads it to infer checkpoint availability. Configure explicit persistent
Notarius chunk-plan and checkpoint roots when reuse must survive separate
Narratio subprocesses. A changed reference, transcript, selected profile,
profile content, endpoint, or other Notarius checkpoint identity input can
produce a legitimate cold miss.
There are two independent resume boundaries: Narratio may skip a completed
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
reuse compatible accepted work only after Narratio has decided extraction must
execute. Deleting Narratio's local reference snapshots does not invalidate a
complete durable extraction; missing snapshots are reconstructed only when a
later extraction actually runs.
## Migrating Manifest Overlay Provenance
The pipeline-selection rename is a clean cutover. Manifests containing
`effective_config.selected_profile` are rejected by strict decoding; Narratio
does not silently discard or translate that retired field.
Before using an affected local session with the new version:
1. Stop Narratio activity for the session and back up its manifest.
2. Rename only `effective_config.selected_profile` to `selected_overlay`, keeping
its `name` and `source`, or remove the retired informational member. If a
current `selected_overlay` member already exists, retain it and remove the
retired member rather than creating duplicate keys.
3. Preserve the schema version, digest, stage records, checkpoint evidence,
output references, and other session state. Retry the intended command.
This metadata is informational, not resume authority. Do not delete session
progress or force all stages merely to perform this migration. Other unknown
fields still require investigation rather than blanket removal.
Do not edit immutable remote committed snapshots. Restoring an older snapshot
with retired provenance requires a compatible previous Narratio version, followed
by migration of the restored local manifest before using the new version.
## Migrating Audita and Provider Credentials
This is a breaking clean cutover to Audita v2. Narratio does not translate an
Audita v1 configuration, retain old field aliases, or generate a downstream
configuration. Update both the deployed Audita executable/configuration and
the Narratio pipeline before running polish. Retired Narratio fields are
rejected by strict configuration loading; account for this break when choosing
a release version.
| Previous Narratio setting | New ownership and selection |
| --- | --- |
| `audita.base_url` | Audita v2 `backends.<id>.endpoint` |
| `audita.total_llm_concurrency`, `proposal_llm_concurrency`, `validation_llm_concurrency` | Audita v2 `backends.<id>.concurrency` and optional `queue_capacity`; backend capacity is shared across roles |
| `audita.model`, `validation_model` | Audita PromptKit profile files; optionally select them from Narratio with `audita.proposal_profile` and `validation_profile` |
| `audita.llm_api_key` | `audita.credentials: [PROVIDER_KEY_NAME]`, matching each Audita backend's `credential` name |
| `notarius.llm_api_key` | `notarius.credentials: [PROVIDER_KEY_NAME]`, matching references in Notarius-owned profiles |
| `audita.work_dir_retention`, `report` | Remove; Audita's external configuration owns diagnostics and reporting |
An Audita configuration can also be selected through `audita.config_path`,
and `audita.overlay` selects an Audita overlay independently of Narratio's
`--overlay`. Narratio forwards no implicit provider key: a single provider-named
secret source may be listed for both adapters, and an Audita-only backend key
may be listed only for Audita. Audita resolves credentials for all configured
backends, including those not currently selected. Notarius looks up its own
profiles; Narratio does not discover their credential names. Credential lists
may be cleared with `[]` in a Narratio overlay.
Prepare the external file from Audita's maintained
[local example](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/examples/minimal-config.yml),
[service example](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/examples/production-config.yml),
and [v2 migration guide](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/migrations/v2.md).
Before switching the executable, validate the file offline:
```bash
audita config validate --config /usr/local/etc/audita/config.yml
```
Use `audita config print-effective --config /usr/local/etc/audita/config.yml --view profiles`
to inspect downstream profile selection without contacting a provider. For a
production Audita configuration, leave ordinary reports enabled
and debug disabled unless an operator deliberately chooses otherwise; Narratio
does not override those settings. Audita writes `errors.yml`, `recoveries.yml`,
and `report.md` beside the polished output when reporting is enabled. An
exit-zero run with a valid transcript succeeds even if those reports describe
recoverable errors or no applied corrections. See the [Audita integration
contract](./integrations/audita.md) for report placement and limits.
Narratio's polish resume evidence tracks selected config path, Audita overlay,
profiles, ordered modules, description, schema, and output identity. Editing
Audita-owned files behind unchanged selectors is invisible to Narratio; force
polish with `narratio run-stage polish <session-id> --force` after such edits.
## External Command Lifecycle
When an external command is cancelled or times out, Narratio terminates its
@@ -311,6 +503,9 @@ tree and reports which stream exceeded the limit.
Run-local diagnostics are:
- `runs/{run_id}/polish/outputs/transcripts/errors.yml`,
`recoveries.yml`, and `report.md` when Audita writes them beside its
run-local polished transcript
- `runs/{run_id}/extract/notarius.receipt.json`
- `runs/{run_id}/extract/notarius.stderr.log`
- `runs/{run_id}/extract/notarius-output/` before durable promotion
@@ -324,6 +519,10 @@ Unlisted files and unsafe entries are never uploaded; a run with no declared
current-run diagnostics uploads no run-local files. No distinguished diagnostic
file is uploaded. The durable bundle is never scanned for implicit publication;
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
To inspect a published polish report remotely, read its declared object under
the run archive's `polish/outputs/transcripts/` prefix. Session restore does
not restore run-archive diagnostics as canonical session files; it restores only
the committed manifest and declared transcript/artifact objects.
To intentionally replace the current extraction result, run:
@@ -334,13 +533,13 @@ narratio run-stage extract 2026-04-04 --force
Narratio automatically reruns extraction when its recorded invocation contract,
prepared Narratio reference identities, or durable output validation changes.
The semantic portion covers Notarius enablement, pipeline identity, declared
reference mapping, and output contracts. Executable, timeout, working directory,
and private config-file paths are operational and do not invalidate a current
result.
It cannot fingerprint configuration files, profiles, prompts, modules, or
reference mapping, output contracts, and the explicit Notarius configuration
profile selector. Executable, timeout, working directory, and private
config-file paths are operational and do not invalidate a current result.
It cannot fingerprint configuration or profile contents, prompts, modules, or
other references loaded transitively by Notarius itself. Force extraction after
changing any of those inputs, even when the top-level Narratio and Notarius
config paths remain the same. A forced extract
changing any of those inputs behind an unchanged selector, even when the
top-level Narratio and Notarius config paths remain the same. A forced extract
marks successful downstream stages stale. Ordinary extraction failures or
outcome changes also stale affected downstream stages, while an identical
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
@@ -506,6 +705,7 @@ Durable session paths:
- `logs/**`
- `config/**`
- `runs/**`
- `cache/notarius/references/<sha256>/<prepared-basename>`
`manifest.json` is the versioned session-progress document. Narratio accepts
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
@@ -516,6 +716,13 @@ relevant workflow requires.
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
stderr, and pre-promotion output remain in the producing run's `extract`
directory as described in [Extraction Workflow](#extraction-workflow).
Stable reference snapshots live in the session cache path above. Ordinary
published-run cleanup removes its selected run-local target but retains this
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
or digest directory manually only while no command is active for the session;
regular missing or corrupt content is reconstructed on the next executed
extraction, while symlinks, directories, and other unsafe conflicts must be
investigated and removed deliberately.
Run-local layout:
@@ -550,18 +757,29 @@ converge on `0664`. Narratio explicitly applies these modes so a restrictive
caller umask does not remove group write or setgid. It does not change file or
directory ownership: the configured workspace's existing group is inherited.
Windows does not implement POSIX mode bits or setgid semantics. Configure the
workspace, spool, and cache locations with an ACL that grants the collaborating
group read/write access, and configure credential locations with an ACL limited
to the intended credential owner. Do not use POSIX mode displays as evidence of
Windows access control.
API keys are credentials, not ordinary workspace data. Store them outside the
shared workspace or in a separately restricted credential location; ordinary
workspace group access must never be treated as authorization to read keys.
On POSIX, provision a credential directory as `0700` and credential files as
`0600`; Narratio rejects group- or other-readable configured credential paths.
On Windows, restrict the directory and files with ACLs to the credential owner.
Provision a credential directory as `0700` and credential files as `0600`;
these permission modes are an operator requirement and are not enforced by
ConfigKit.
Every pipeline-loading command bootstraps credentials once, before the first
configuration load and before any adapter is constructed. `--secrets-dir`
selects an explicit directory (relative paths use caller CWD). Otherwise a
present `secrets/` beside the resolved root pipeline is used, while an absent
conventional directory is skipped. Explicit missing/empty selections and
invalid existing directories fail the command. ConfigKit follows a symlink used
as the selected directory, but immediate entries must be regular non-symlink
files. Existing environment variables—including empty values—always win.
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
environment references. When both are absent, the AWS SDK retains its default
credential provider chain. When both are configured, Narratio supplies their
load-time resolved values through an explicit static provider. Audita and
Notarius credential lists likewise resolve at pipeline load. Each adapter
receives only its listed names and captured values, with no alias mapping or
ambient environment inheritance. See [configuration](./config.md#audita-selection-and-credentials).
External adapter results are individually bounded before Narratio validates or
materializes them. These per-file limits do not reserve disk space: prevent hard
@@ -624,4 +842,5 @@ normal migration path.
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
- Restore requires configured object storage and committed remote current state.
- Storage-backed commands load filesystem secrets before object-store initialization.
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
resolution and object-store initialization.

95
docs/policy/agents.md Normal file
View File

@@ -0,0 +1,95 @@
# Agent Policy
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
## Purpose
Agents working in this repository must read and adhere to the following house rules.
## Implementation Discipline
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
request. Do not implicitly strengthen requirements, restrictions, or guarantees
beyond what the user actually requested.
Simplify the mechanism, not the user's choices.
**Keep implementation machinery proportionate.** Use straightforward,
idiomatic code and introduce only the machinery needed to support the requested
behavior and flexibility. Optimize for conceptual simplicity and
maintainability, not for the fewest lines, fewest options, or narrowest product
surface.
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
harden, or otherwise narrow behavior unless required by the user, an explicit
repository contract, or an unavoidable correctness constraint. Preserve
operator and caller choice where policy has not been established. If an
additional restriction appears advisable, present it as a decision rather than
silently implementing it.
A correctness constraint must follow from the requested behavior or an
established invariant, not from the convenience of the chosen implementation.
Explain the specific failure it prevents and why a less restrictive
implementation cannot satisfy the contract.
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
or invariants beyond those expressly requested by the user.
**Use the standard library.** For example: in almost every case, the best
approach to copying files is to use the standard library. Writing custom code to
implement power-loss protection at the application level, or to guard against
bad actors with privileged and concurrent access to the host's filesystem,
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
idiomatic approach unless another path is expressly requested by the user.
**An implementation detail is not repository policy.** Tests and documentation
must not elevate incidental behavior into a durable guarantee or restriction
unless that behavior was expressly selected as part of the contract. Existing
explicit contracts remain authoritative; changing or weakening one requires a
deliberate decision rather than being inferred from this principle.
Agent-authored plans, tests, documentation, and prior implementations are not
evidence that an additional restriction was deliberately approved. Identify the
authorizing requirement or decision; when that authority is unclear, surface
the ambiguity rather than treating repetition as approval.
Standard-library guidance does not authorize weakening Narratio's established
filesystem, manifest, or publication contracts. Preserve them unless a deliberate
decision changes them.
## Git Commit Policy
By default, create a commit after completing each coherent unit of work, using a
concise, plain-English message describing what changed. This policy is standing
authorization to create those local commits; a separate user request is not
required. An explicit user instruction to leave work uncommitted takes precedence.
A unit of work is a focused, reviewable outcome, including its relevant tests and
documentation. Do not commit after every intermediate edit or create empty commits.
Complete the appropriate validation before committing. If work is incomplete or
validation is blocked, report that state rather than presenting it as completed.
Before committing:
1. Run `git status --short` and review the relevant diff, including any staged
changes. Identify changes that predate the current unit of work.
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
only when every included change belongs to that unit. Do not disturb unrelated
staged or unstaged work.
4. Review `git diff --staged` and run appropriate checks for the change. Do not
repeat checks already passed for unchanged content merely to create a commit.
5. Review `git log --oneline -n 10` and use
`git commit -m "<concise plain-English commit message>"`.
6. Report the commit identifier and any remaining uncommitted work.
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
and explain the overlap rather than silently including that work. If a commit
fails, preserve the changes and report the failure; do not bypass hooks.
Default commit authorization does not authorize pushing, publishing, amending,
resetting, reverting, or rewriting history. Those actions require a separate user
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
staging and validation discipline above.

View File

@@ -1,5 +1,9 @@
# Architecture
**Base Policy Version:** 1.0
**Last Updated**: 2026-09-25
This document defines Narratio's intended high-level architecture and the
invariants that changes must preserve. Implemented component details belong in
the [Internal Overview](../internal/overview.md) and its linked documents.
@@ -146,15 +150,15 @@ Configuration is strict, explicit, centralized, and operator-oriented.
- YAML decoding rejects unknown fields.
- Defaults are centralized and testable.
- Empty configured values do not silently replace meaningful defaults.
- Validation rejects invalid composition before stage execution where
- Validation rejects invalid configuration before stage execution where
practical.
- Root-owned imports and one selected profile resolve deterministically through
- Root-owned imports and one selected overlay resolve deterministically through
the configuration owner; commands do not implement their own merge rules.
- Canonical party rosters are campaign-owned. Their derived players projection
and concrete character-family artifacts are resolved before runtime stages
or adapters receive configuration.
- Resume uses stage- or artifact-owned semantic evidence for observable
result-affecting configuration; profile identity and an effective digest are
result-affecting configuration; overlay identity and an effective digest are
provenance, never blanket cache keys.
- Session templating remains narrow and deterministic rather than becoming a
general configuration language.
@@ -234,20 +238,26 @@ configuration, logs, reports, diagnostics, and Notarius artifacts—is
intentionally shareable with the configured workspace group. API-key material
is sensitive and is not covered by the ordinary workspace-sharing policy.
On POSIX systems, Narratio-created ordinary workspace directories converge on
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
a cross-build gate and release asset, but no native runtime release gate.
Windows and macOS amd64 are unsupported and have no compatibility promise or
release asset.
On supported systems, Narratio-created ordinary workspace directories converge on
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
umask is restrictive. This preserves the existing workspace group for nested
creation and atomic replacements without changing ownership. API-key storage
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
are not authoritative; operators must provide the equivalent shared-group and
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
uses a separate restrictive contract.
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
to manifests, logs, generated configuration, reports, publish metadata,
documentation, or examples. Secrets enter through configured environment
variable names or secret-file references. Diagnostics should avoid transcript
and prompt content unless a deliberate, bounded inspection mechanism requires
it.
documentation, or examples. Pipeline configuration names environment variables;
command-owned startup may populate previously unset variables from ConfigKit's
explicit or conventional secrets directory. This bootstrap occurs once per
invocation before any typed configuration load or adapter construction and is
not hidden inside library loaders. Safe bootstrap diagnostics may identify
files and statuses but never values. Diagnostics should avoid transcript and
prompt content unless a deliberate, bounded inspection mechanism requires it.
Logs, reports, generated invocation files, generated configuration, and render
debug files are diagnostics, not canonical pipeline products. They should be

View File

@@ -1,5 +1,9 @@
# Documentation Policy
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
## Purpose
This policy assigns each documentation topic to one canonical owner. Its goal is
@@ -59,7 +63,9 @@ secret values.
| --- | --- | --- | --- |
| Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
| Maintainer release procedure | `docs/release.md` | Version selection, candidate preparation and validation, guarded tag publication, release completion boundary, failure recovery, and optional asynchronous inspection. | Script implementation mechanics, current application contracts, and historical release summaries. |
| Agent policy | `docs/policy/agents.md` | Agent implementation discipline, authority boundaries, and local commit policy. | Application contracts, architecture, testing policy, and contributor procedures. |
| Release policy | `docs/policy/release.md` | Versioning, release integrity, documentation and validation requirements, immutability, publication controls, and explicit exceptions. | Repository-specific commands, installation, historical summaries, and CI implementation detail. |
| Maintainer release procedure | `docs/release.md` | Prerequisites, concrete candidate validation and publication commands, deliverables, failure recovery, and optional consumer-facing inspection. | General release requirements and exceptions, current application contracts, and historical release summaries. |
| Historical release summary | `docs/releases/<tag>.md` | Immutable summary, compatibility, upgrade, and changes for one released version. | Current maintainer procedure and current application contract details. |
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
@@ -100,6 +106,14 @@ configuration, operational, or integration contract. When a workflow crosses
these topics, choose the document that owns the task and link to the other
contracts.
### Releases
[Release policy](release.md) owns requirements and explicit exceptions;
[the release procedure](../release.md) owns the commands used to satisfy them.
Release notes record history at an immutable tag. Neither the procedure nor a
release note may silently weaken policy or substitute for updates to current
canonical documentation.
### Contracts And Implementation
Integration and API documents define externally observable shapes and

221
docs/policy/release.md Normal file
View File

@@ -0,0 +1,221 @@
# Release Policy
**Base Policy Version:** 1.0
**Last Updated**: 2026-09-25
## Purpose
This policy defines the minimum requirements for selecting, validating,
publishing, verifying, and correcting releases. Its goal is to make every
release intentional, reproducible from an exact source revision, documented
for consumers, and safe from retrospective replacement.
The repository-specific commands that implement these requirements belong in
`docs/release.md`. Historical release summaries belong in `docs/releases/`.
Neither document may weaken this policy, including its explicit repository
exceptions.
## Release Model
A release consists of:
- one exact source commit reachable from the repository's primary branch;
- one immutable version tag identifying that commit;
- one version-matched, checked-in release note; and
- every artifact, image, package, registry entry, or other deliverable the
repository declares for that release.
The repository's primary branch is `main` unless an adopted policy documents
an exception. A release candidate must be committed and pushed to that branch
before its release tag is published.
The source tag is the authoritative release identity. Published deliverables
must be traceable to the tagged source and must agree with its version. A
repository may publish source only. When it publishes additional deliverables,
its release procedure must enumerate them and define how each is built,
identified, published, and verified.
Release automation may validate or publish a release only when its exact
responsibility is documented. Automation must not silently change the source
candidate, move a published tag, replace a published deliverable, or repair a
release in place.
## Versioning And Compatibility
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
no leading zeroes. Each version must be unused locally and in every configured
remote or publication channel before publication begins.
Unless an adopted policy explicitly supports and defines prereleases,
prerelease versions are not permitted.
For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and
- a patch release may make backward-compatible fixes.
Before `v1.0.0`, a minor release may change a documented public, operational,
configuration, data, integration, or compatibility contract when the release
note explains the impact and required consumer action. A patch release must
not intentionally break those contracts within its minor line.
A repository with additional compatibility promises must define them in its
adopted policy or canonical contract documentation. A release note may explain
a compatibility change, but it must not redefine the current compatibility
contract.
## Release Documentation
Every release made under this policy must include
`docs/releases/<version>.md` in the tagged source tree. The filename must use
the complete version tag, and the document must begin with a title containing
the product name and that exact version.
Every new release note must contain these sections:
```markdown
# <Product> vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
Use an explicit statement such as "No action required" when a section has no
substantive content.
Release notes are immutable historical summaries. They should link to current
canonical documentation for exact behavior. Every affected canonical document
must be updated in the same candidate; a release note must not substitute for
current documentation or defer required documentation changes.
## Candidate Preparation And Validation
Each repository must provide one documented release-check command or script
that exercises the complete set of automated source-candidate gates. The
repository-specific procedure must state all additional manual gates.
The required gates are:
- dependency or module hygiene appropriate to the language and ecosystem;
- the complete deterministic, offline default test suite;
- static analysis and linting used by the repository;
- formatting and whitespace checks;
- validation of maintained examples, configuration, schemas, generated
sources, and documentation commands where applicable;
- review of changed local documentation links;
- removal of obsolete roadmap documents under `docs/roadmap/` that describe
completed work included in the release; and
- review for unintended files, generated output, credentials, private data,
and unrelated changes.
## Publication Guards
Immediately before publication, the maintainer or release automation must
fetch the current primary branch and tags from the authoritative remote and
verify all of the following:
- the version has the required format and is not already published;
- the current branch is the primary release branch;
- the worktree and index are clean;
- the current commit is the recorded, validated candidate;
- the authoritative remote branch points to that same commit;
- the matching release note exists in that commit and has the required title
and sections;
- all required approvals and automated gates succeeded for that commit; and
- no local or remote tag already uses the selected version.
Create the version tag against the guarded commit and verify its target before
publishing it. Publish only the exact intended ref and deliverables. Broad
operations such as `git push --tags` are discouraged because they can publish
unreviewed refs unrelated to the release.
The repository-specific procedure must make these guards executable or give
copyable commands for performing them.
## Immutability And Correction
Published version tags and published deliverables are immutable. Do not move,
overwrite, recreate, or delete a published tag. Do not replace an artifact,
image, package, checksum, signature, provenance statement, or release note
under an already published version.
If validation fails before publication, correct the candidate, rerun the
complete gates, and repeat the publication guards. An unpublished local tag
may be removed after confirming that it was never published or used by a
release channel.
If publication or post-publication verification reveals a defect, leave the
published release intact. Correct the defect on the primary branch, choose a
new patch version or other semantically appropriate new version, write a new
matching release note, and perform the complete release procedure again.
A distribution channel's supported yank, deprecation, or advisory mechanism
may warn consumers away from a defective release, but it must not replace that
release's contents or reuse its version.
## Repository Release Procedure
Every adopting repository must maintain `docs/release.md` as the canonical
maintainer runbook for implementing this policy. It must define, in execution
order:
1. maintainer prerequisites and required access;
2. version selection and compatibility review;
3. release-note creation;
4. candidate preparation and the complete validation command;
5. candidate commit, push, and exact commit recording;
6. publication guards;
7. exact tag and deliverable publication commands;
8. authoritative remote and consumer-facing verification; and
9. failure handling and correction.
The procedure should use copyable, fail-fast commands where practical. It must
name the primary branch, authoritative remote, tag form, published
deliverables, supported build targets, release automation responsibilities,
and consumer verification path. It must link to this policy instead of
restating or redefining its requirements.
## Policy Exceptions
### Tag-push completion and asynchronous deliverables
Narratio's automated release responsibility ends when the exact intended tag is
successfully pushed to `origin`. It must not wait for, poll, or check CI/Gitea
results, download assets, or verify their checksums as a completion condition.
Binary and checksum publication is asynchronous; consumer-facing inspection is
optional follow-up, not a gate on the source release.
This deliberately qualifies the base policy's complete-deliverable and
post-publication verification requirements. CI reliability is outside the
release command's scope; an unavailable publication service must not cause an
automated releaser to wait indefinitely or claim to repair a published version.
Compensating controls are complete local candidate validation (including supported
cross-builds and a host version check), exact source/tag guards, immutable source
tags, asynchronous build validation, and non-overwriting asset publication.
Successful tag publication is not evidence that binary assets are available.
Optional inspection and corrective-release procedures belong in
[the release procedure](../release.md).
### Authoritative publication scope
Narratio's synchronous publication guards inspect the authoritative `origin`
remote and its `main` branch, not every configured Git remote or a Gitea release
entry. The command does not add release-service or registry availability checks.
This qualifies the base policy's all-channel unused-version requirement.
Other remotes are not Narratio release authorities, and Gitea asset publication
is asynchronous. The compensating controls are local and origin tag uniqueness,
exact candidate matching, and the publisher's non-overwrite settings. Any known
already-published version or conflicting deliverable remains ineligible for
reuse: choose a new version rather than bypassing guards. A future additional
distribution channel requires deliberate policy/procedure updates.

View File

@@ -1,5 +1,9 @@
# Testing Policy
**Base Policy Version:** 1.0
**Last Updated**: 2026-09-25
## Purpose
Our tests exist to make **incorrect changes expensive and correct changes cheap**.

View File

@@ -1,71 +1,97 @@
# Releasing Narratio
This document is the maintainer procedure for creating a Narratio source and
binary release. The synchronous release boundary is a successful push of one
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
change that result.
This is the maintainer runbook for the [Release Policy](policy/release.md).
Its [explicit exceptions](policy/release.md#policy-exceptions) preserve the
successful tag push as the automated responsibility boundary. Binary publication
is asynchronous; later consumer-facing inspection is optional.
## Prerequisites and publication scope
Use a checkout on `main` with Git authentication permitting fetches and pushes
to `origin`, the authoritative Git remote. Install Git, a POSIX shell, standard
Unix build tools, and the Go toolchain below. Make required modules available
before offline validation. No model/provider credentials are needed. Gitea
publication credentials belong to CI's secret store, not the local command.
Synchronous guards inspect local tags and `origin`, not every configured remote
or Gitea release entry. Do not select a version known to have been published in
any channel, even if these guards cannot see it.
## Go prerequisite
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
asset construction; that asynchronous work does not change the tag-push
completion boundary.
uses the explicitly pinned `golang:1.26.7` image for validation and asset builds.
That asynchronous work does not change the tag-push completion boundary.
## Choose a version and write its note
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
`vMAJOR.MINOR.PATCH`:
Select an unused stable version under the
[versioning rules](policy/release.md#versioning-and-compatibility). Narratio is
past v1.0.0; intentional CLI/configuration breaks require a major release.
- increment `MINOR` for backward-compatible features;
- increment `PATCH` for backward-compatible fixes; and
- reserve a new `MAJOR` for an intentional breaking documented contract.
Create `docs/releases/vMAJOR.MINOR.PATCH.md` using the
[required note structure](policy/release.md#release-documentation).
Describe CLI, configuration, artifact, integration, and operating-contract
changes, and the required operator action. Link to current canonical
documentation for exact behavior.
Before preparing the candidate, create
`docs/releases/vMAJOR.MINOR.PATCH.md` with this structure:
## Prepare and record the candidate
```markdown
# Narratio vMAJOR.MINOR.PATCH
Before committing, perform these manual reviews in addition to automated checks:
This release ...
- Confirm compatibility/version selection and update affected canonical
documentation and maintained examples.
- Remove completed roadmap documents covering work included in this release;
retain pending work. Their history remains in Git. Do not delete historical
release notes.
- Review the diff, staged changes, and untracked files for credentials, private
data, generated output, unintended files, and unrelated changes.
- Review changed documentation links and behavior-bearing commands. Automated
link checks supplement rather than replace review.
## Summary
Commit the note and candidate changes, then push that commit to `main` through
the ordinary development workflow. Record the exact candidate:
## Compatibility
## Upgrade
## Changes
```sh
git rev-parse HEAD
```
The compatibility section identifies relevant CLI, configuration, artifact,
integration, or operating-contract changes. The upgrade section states the
required operator action, or explicitly says that no special action is
required. Release notes are immutable historical summaries; link to the
current canonical documentation for detailed behavior.
Commit the note and all candidate changes, then use the ordinary development
workflow to push that commit to `main`. Do not create a release tag before the
candidate is committed and `origin/main` contains the exact same commit.
Do not create a tag until `origin/main` points to that same commit. If the
candidate changes, repeat the reviews and validation.
## Validate the candidate
Run the shared checker from any directory:
From the repository root, run:
```sh
scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
```
The script also works when invoked by its absolute path from another directory.
It validates the version and matching note, module hygiene, formatting,
whitespace, uncached tests, race tests, static checks, documentation, examples,
and six official cross-build assets. It uses `GOWORK=off`, does not contact
application services, CI, or Gitea, and does not create tags or modify tracked
source. Fix any failure on `main`, commit it, push it normally, and rerun the
checker.
and three cross-build assets. It uses `GOWORK=off`, does not contact application
services, CI, or Gitea, and does not create tags or modify tracked source.
Fix failures on `main`, commit, push normally, and rerun validation.
The checker builds Linux, macOS, and Windows assets for `amd64` and `arm64`.
Cross-builds prove compilation; they are not native macOS or Windows runtime
evidence.
The macOS cross-build proves compilation, not native runtime behavior.
## Declared deliverables and automation
The source release is the exact version tag at the validated commit.
The tag-triggered `.woodpecker/release.yml` validates the tagged source,
runs `scripts/build-release-assets.sh`, and publishes these Gitea assets:
- `narratio-vMAJOR.MINOR.PATCH-linux-amd64`
- `narratio-vMAJOR.MINOR.PATCH-linux-arm64`
- `narratio-vMAJOR.MINOR.PATCH-darwin-arm64` (best effort)
- `SHA256SUMS`
The builder embeds the version, disables ambient Go workspaces, and checks a
host binary's `version` output. The release publisher uses the matching
checked-in note and generates SHA-256 checksums. Existing files are skipped and
overwrite is disabled. No image or package-registry deliverable is declared.
CI does not choose versions, modify source candidates, or move tags.
## Publish the tag
@@ -78,27 +104,64 @@ scripts/release.sh vMAJOR.MINOR.PATCH
The command fetches and checks `origin/main`, re-runs candidate validation,
then fetches and checks again before creating an explicitly unsigned lightweight
tag for the originally recorded commit. It refuses dirty, divergent, changed,
or already-tagged candidates. It pushes only:
or already-tagged candidates. It verifies the local tag target and pushes only:
```text
refs/tags/vMAJOR.MINOR.PATCH:refs/tags/vMAJOR.MINOR.PATCH
```
It never commits changes, pushes `main`, force-pushes, moves a tag, or pushes
all tags. A successful push of that exact ref completes the release command;
the command prints the tag and commit, then returns without waiting for CI,
querying Gitea, downloading assets, or checking checksums.
It never commits changes, pushes `main`, force-pushes, moves tags, or pushes all
tags. Successful publication of that ref completes the command; it prints the
tag and commit and returns without waiting for CI, querying Gitea, downloading
assets, or checking checksums.
If a failure occurs before the tag is created, correct the candidate on `main`
and repeat validation. If the push fails after local tag creation, the local tag
is intentionally retained for inspection and the command must not be retried
blindly. Once the upstream tag has been pushed, it is immutable. Correct any
defect or failed asynchronous publication with a new patch version, a new
release note, and the complete procedure again.
## Failure handling
Before tag creation, correct the candidate on `main` and repeat validation.
If a push fails after local tag creation, the local tag is retained for inspection.
Do not retry blindly or assume the remote received nothing: a connection failure
can leave publication uncertain. Inspect the exact remote ref before taking any
local-tag cleanup action. This is recovery from a failed command, not a
post-success publication gate.
After publication, follow the
[immutability and correction rules](policy/release.md#immutability-and-correction).
Correct defects or failed asynchronous publication with a new semantically
appropriate version, matching note, and the complete procedure. Never replace
existing tagged assets in place.
## Optional asynchronous inspection
After a successful tag push, a human may later inspect the tag-triggered
Woodpecker run and the corresponding Gitea release for binaries and checksums.
This is optional follow-up only. Automated releasers must not wait for, poll,
or treat CI/Gitea completion as a condition of the successful tag push.
A human may later inspect the tag-triggered Woodpecker job and Gitea release.
This is optional follow-up, never part of the automated releaser's completion
criteria. Do not wait for or poll CI/Gitea from the release command.
For optional source verification, substitute the released version below and
compare the remote target to the recorded candidate hash:
```sh
git ls-remote --tags origin refs/tags/vMAJOR.MINOR.PATCH
git rev-parse 'refs/tags/vMAJOR.MINOR.PATCH^{commit}'
```
Download the three named binaries and `SHA256SUMS` from the matching Gitea release
into an empty inspection directory. In that directory, verify on Linux:
```sh
sha256sum --check SHA256SUMS
```
On macOS, use `shasum -a 256 --check SHA256SUMS`. After successful verification,
make the matching host binary executable and run its version command, for example
on Linux amd64 (replace the version in both commands):
```sh
chmod +x ./narratio-vMAJOR.MINOR.PATCH-linux-amd64
./narratio-vMAJOR.MINOR.PATCH-linux-amd64 version
```
Expected output is `narratio vMAJOR.MINOR.PATCH`. Use only the matching architecture
on other supported hosts. Checksums establish consistency with the downloaded
checksum file, not a new signature or independent authenticity guarantee.
Missing assets are an asynchronous publication problem, not grounds to move or
reuse the successfully published source tag.

View File

@@ -22,5 +22,7 @@ See the [release procedure](../release.md) for creating a candidate and tag.
When asynchronous publication succeeds, the corresponding Gitea release is the
canonical source for downloadable binaries and checksums.
- [v2.0.0](v2.0.0.md)
- [v1.7.0](v1.7.0.md)
- [v1.6.0](v1.6.0.md)
- [v1.5.0](v1.5.0.md)

94
docs/releases/v2.0.0.md Normal file
View File

@@ -0,0 +1,94 @@
# Narratio v2.0.0
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
simplifies the supported release platforms, and makes artifact definitions and
their effective dependency graph directly inspectable.
## Summary
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
imports, fixed-path named profiles, provenance, defaults, and typed credential
references. The recommended production layout keeps production values in the
base, defines each ordinary artifact and artifact family in its own imported
file, and keeps testing changes in `profiles/testing.yml`.
Configured-artifact inputs now create their dependency edges automatically.
`narratio config artifacts` reports the finalized concrete catalog, including
generated party-family members, without loading a session or initializing
runtime adapters.
## Compatibility
This is a breaking configuration and platform release. There is no runtime
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
presence/default behavior. Defaults apply only to absent values; explicit empty,
zero, false, and null values retain their typed meaning and may fail Narratio
domain validation.
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
and macOS amd64 builds are no longer produced or supported. Existing v1 release
notes remain historical descriptions of their releases.
Inspection commands still avoid session, manifest, workspace, adapter, and
remote-service activity, but they are not credential-free loaders: every
explicit typed credential reference in the selected effective configuration must
resolve; references replaced or removed by that profile need not resolve.
Pipelines containing artifact families also require a campaign so the canonical
party can be expanded.
`config show` is a secret-safe review projection, not a promised reloadable
authoring file.
## Upgrade
Migrate each operational bundle before running v2:
1. Replace old composition syntax with root-owned
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
additive base sources; they do not override one another.
2. Fold the old production overlay into the base pipeline. Move testing changes
to `profiles/testing.yml`; compare the two independent resolutions with
`narratio config diff default testing`.
3. Follow the maintained
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
stable artifact identity.
4. Move credential files to an untracked `secrets/` directory beside
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
5. For static S3 credentials, configure both S3 references (commonly
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
To use instance roles, workload identity, shared SDK files, or another AWS SDK
discovery provider, omit both references. A partial pair is invalid.
6. Remove a `depends_on` entry when it only duplicates a
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
without supplying the upstream output as an input.
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
that previously acquired defaults after being explicitly empty. Do not use
`config show` output as a source migration mechanism.
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
with the same profile, campaign, and secrets selection intended for runtime.
Resolve the campaign-owned party whenever families are present.
9. Update deployment and download automation to one of the three maintained
platform assets before removing the v1 binary.
The [configuration reference](../config.md), [CLI reference](../cli.md), and
[maintained examples](../../examples/README.md) are the canonical detailed
guides.
## Changes
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
while keeping Narratio domain validation and post-campaign expansion explicit.
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
unified safe effective show/diff/digest projection, and source provenance.
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
Audita credential references without exposing resolved values.
- Inferred, normalized, and validated configured-artifact input dependencies as
part of the one concrete graph consumed by analyze planning and resume logic.
- Added `config artifacts` with stable concrete identities, definition sources,
effective dependencies, and generated family/party provenance.
- Added the maintained production-base/testing-overlay artifact authoring layout.
- Reduced release assets and active platform support to Linux amd64/arm64 and
macOS arm64.

70
docs/releases/v2.1.0.md Normal file
View File

@@ -0,0 +1,70 @@
# Narratio v2.1.0
This release adds explicit Notarius credential forwarding and cache reuse,
with predictable forced regeneration and clearer artifact-development guidance.
## Summary
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
its accepted execution checkpoints. Stable, verified reference snapshots allow
unchanged references to retain their cache identity across Narratio invocations.
Configuration loading now uses ConfigKit v0.3.0.
## Compatibility
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
fields are optional. Existing v2 configuration needs no schema migration.
Configured credential references must resolve during configuration loading,
even when extraction will not execute; omitted references add no requirement.
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
refresh. This includes `regenerate-artifacts`, which remains an alias for a
forced run from extract through analyze. `analyze` continues to force analysis
only, preserving extraction results and transcripts.
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
minimum-version gate. The executable must support the flags selected for the
invocation, including `--chunk_cache refresh` for forced extraction. Reference
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
schemas are unchanged; reused producers may report zero production attempts.
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
arm64. macOS validation is cross-compilation, not native runtime testing.
## Upgrade
No special migration is required for existing v2 configuration. To use the new
features:
- Declare `notarius.llm_api_key` with the environment-variable name expected by
the deployed Notarius profile, supplying its value through the existing
environment or secrets-directory mechanism.
- Enable checkpoint recording in Notarius configuration before setting
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
roots for service deployments. An optional `notarius.chunk_cache` overrides
its ordinary chunk-plan policy.
- Expect an initial checkpoint miss when moving from old run-local reference
paths to stable session snapshots. No migration or deletion of old caches is
required. Force refreshes the current input's plan, not the whole cache.
- Use `narratio analyze SESSION` for analysis-only iteration, and
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
See [Configuration](../config.md), the
[Notarius integration contract](../integrations/notarius.md), and
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
for current details.
## Changes
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
from machine configuration.
- Added optional typed Notarius credentials, exact-name environment delivery,
safe configuration projections, and captured-stream redaction for custom names.
- Added Notarius resume and chunk-cache controls with force taking precedence.
- Added verified, content-addressed session reference snapshots that survive
ordinary run cleanup without becoming published artifacts.
- Preserved relative-workspace support for snapshot delivery and confinement.
- Kept credentials and cache-policy controls outside extract's semantic resume
identity, so operational changes alone do not invalidate completed results.
- Added an opt-in real Notarius integration test using a local provider to
verify reuse, partial-failure recovery, and forced refresh without paid calls.

33
docs/releases/v2.1.1.md Normal file
View File

@@ -0,0 +1,33 @@
# Narratio v2.1.1
This release fixes configuration loading for publish rules that explicitly set
requiredness while relying on a derived destination.
## Summary
Publish rules with a derivable source and omitted `dest` now load correctly
whether `required` is omitted, explicitly true, or explicitly false. Previously,
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
diagnostic instead of accepting valid configuration.
## Compatibility
This is a backward-compatible bookkeeping correction. Publish schemas,
destination derivation, requiredness defaults, and YAML syntax support are
unchanged. Explicit destinations and explicit false values remain intact.
No ConfigKit upgrade or publication/storage behavior change is involved.
## Upgrade
No special upgrade action or configuration migration is required. Operators
need not add a derivable `dest`, remove explicit `required` values, or change
flow-style YAML to block-style YAML. See the
[configuration reference](../config.md#publish-configuration-summary).
## Changes
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
while retaining field-level resolution records.
- Added real load/validation and safe effective-configuration regression coverage
for built-in and configured-artifact sources, explicit requiredness, and an
explicit destination that must remain unchanged.

26
docs/releases/v2.1.2.md Normal file
View File

@@ -0,0 +1,26 @@
# Narratio v2.1.2
This release allows secrets directories to contain hidden metadata without
preventing credential loading.
## Summary
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
valid credentials from loading and are omitted from bootstrap reports.
## Compatibility
This is a backward-compatible dependency bug fix. Configuration schemas,
existing-environment precedence, and validation of non-hidden entries are
unchanged. Hidden entries still count toward the directory entry limit.
## Upgrade
No special upgrade action or configuration migration is required. See the
[credential configuration reference](../config.md) for secrets-directory usage.
## Changes
- Updated ConfigKit from v0.3.0 to v0.3.1.
- Clarified hidden-entry handling in the secrets-directory documentation.

53
docs/releases/v2.2.0.md Normal file
View File

@@ -0,0 +1,53 @@
# Narratio v2.2.0
This release adds resumable per-speaker transcription, replayable WhisperX
uploads, and explicit Notarius configuration profile selection.
## Summary
Successfully checkpointed speaker transcripts survive later transcription
failures. Ordinary retry reuses valid completed speakers and processes only the
remaining work. WhisperX uploads can now be replayed by the HTTP transport
without buffering whole audio files.
## Compatibility
Existing CLI commands, speaker identities, transcript formats, and pipeline
ordering are unchanged. Session manifests gain optional per-speaker completion
evidence. Older manifests remain readable, but older Narratio binaries may
reject manifests containing the new fields; downgrading after transcription
is not transparent.
The optional `pipeline.notarius.config_profile` field selects a downstream
Notarius configuration profile independently of Narratio's own profile. Using
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
supporting version. Omitting the field retains downstream default selection.
## Upgrade
No configuration migration is required. When transcribe is next selected for
an older aggregate-only manifest, all current speakers are transcribed once
to establish reusable evidence. Previously unrecorded results from failed runs
are not automatically imported.
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
reuse of prior results; an ordinary retry after a failed forced invocation can
retain replacements already checkpointed by that attempt. See the
[operations guide](../operations.md) and
[Notarius profile configuration](../config.md#notarius-configuration-profile).
## Changes
- Persist validated speaker completions independently of aggregate stage status,
with serialized saves for concurrent work and input/output integrity checks.
- Rebuild aggregate transcript references when speakers are removed or those
references need repair, without retranscribing otherwise-current speakers.
- Keep removed and incomplete speaker results out of versioned merge inputs.
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
service rejection still fails normally.
- Provide fresh streaming multipart bodies for HTTP transport replay while
preserving application retries, cancellation, and resource cleanup. This
addresses the missing-body-replay limitation, not persistent upstream
protocol failures or exactly-once service execution.
- Pass explicit Notarius configuration profiles through the CLI and include
their selection in extraction reuse decisions.

35
docs/releases/v2.2.1.md Normal file
View File

@@ -0,0 +1,35 @@
# Narratio v2.2.1
This maintenance release consolidates checkpoint persistence for transcription
and analysis without changing their operator-facing behavior.
## Summary
Transcribe and analyze now share one internal operation for preparing isolated
checkpoint candidates, saving them, and promoting successfully saved progress
into live state. Each stage retains its own progress model and recovery policy.
## Compatibility
CLI commands, configuration, manifest schemas, artifact formats, and external
integration contracts are unchanged. Speaker concurrency, artifact selection,
force behavior, and ordinary resume semantics are preserved. No dependency
upgrade or new external-tool version requirement is introduced.
## Upgrade
No special upgrade action or configuration migration is required when upgrading
from v2.2.0. See the [v2.2.0 release notes](v2.2.0.md) when upgrading from an
earlier version for the original speaker-checkpoint upgrade considerations.
## Changes
- Replace duplicate analyze/transcribe save-and-promote sequences with a
shared, private checkpoint helper.
- Isolate checkpoint collections from live state, including analyze records
normalized during a transcription checkpoint save.
- Preserve transcription's caller-owned locking and failure handling, and
analysis's working/committed collections and rollback behavior.
- Add focused checkpoint integrity tests while retaining stage and application
recovery coverage.
- Document the shared [checkpoint persistence boundary](../internal/manifest.md#checkpoint-persistence-boundary).

288
docs/roadmap/audita.md Normal file
View File

@@ -0,0 +1,288 @@
# Adopt Audita v2 and ConfigKit v0.4.0
Status: implemented (2026-09-25); release pending.
## Goal
Migrate Narratio's `polish` integration from Audita's former command/configuration
contract to Audita v2.0.0. Keep Narratio responsible for orchestration and Audita
responsible for correction behavior and LLM execution configuration.
Adopt a consistent provider-named credential list for both Audita and Notarius,
so operators define each secret once and explicitly share it with the downstream
applications that need it. This is a clean cutover without backward-compatibility
shims.
Adopt ConfigKit v0.4.0 and use “overlay” consistently for Narratio's selected
pipeline configuration, distinct from downstream LLM model profiles.
Production runs use Audita's default reporting configuration: Narratio passes
neither `--debug` nor `--reporting=false`. Normal reports remain available without
enabling debug diagnostics. Adding debug reporting through Narratio's testing
profile is deferred.
This document defines the intended end state, migration scope, and policy.
The [implementation plan](implementation.md) owns implementation sequencing and
verification steps.
## Baseline and upstream contract
The baseline is Narratio v2.2.1 and the Audita checkout tagged v2.0.0. Upstream
references are Audita's `docs/migrations/v2.md`, `docs/cli.md`,
`docs/integrations/subprocess.md`, and `docs/integrations/output-files.md`,
available in the sibling `../audita/` checkout. Narratio's current contract is
documented in [Integration: Audita](../integrations/audita.md).
The current integration is not compatible with Audita v2:
- It requests `--report-json` by default; that flag has been removed.
- It can pass retired endpoint, model, concurrency, and work-directory-retention
flags.
- It expects one JSON report instead of the normal v2 report bundle.
- Its single forwarded `AUDITA_LLM_API_KEY` has no implicit meaning in v2;
an Audita backend must explicitly reference that environment variable.
- Its semantic fingerprint sorts modules, although Audita treats module order
and repetitions as meaningful.
Audita v2 requires a version-2 configuration file. An explicit `--config` selects
one; otherwise Audita performs its own documented discovery. Backend definitions
own endpoints, credentials, and capacity. PromptKit profiles own models and
per-attempt controls. Audita owns retry/repair policy and operation deadlines.
Narratio's generated invocation YAML is a diagnostic record, not a substitute
for that Audita configuration file.
## Target architecture
Retain the existing subprocess boundary and `polish` stage. Do not introduce a
second correction engine, an in-process Audita dependency, or a generic external
tool configuration framework.
Narratio owns:
- Selection of the executable and the overall invocation timeout/cancellation.
- Transcript and glossary inputs, run-local output placement, validation of the
corrected transcript, and promotion to the existing polished transcript
identity.
- Explicit configuration/profile selections and supported domain overrides.
- Deliberate credential forwarding, secret-safe invocation diagnostics, and
manifest references to results and reports.
- Existing stage completion, resume, force, publish, restore, and cleanup
behavior.
Audita owns its configuration schema and discovery, backend/model configuration,
provider execution, correction sequence, recovery behavior, and report contents.
Narratio does not translate old settings into a generated v2 configuration or
duplicate Audita's complete validation logic.
## Configuration and invocation migration
### ConfigKit and pipeline overlays
Upgrade Narratio's ConfigKit dependency from v0.3.1 to v0.4.0. ConfigKit's
selected-configuration terminology changes from profile to overlay: the Go API
uses `Options.Overlay`, `LayerOverlay`, and `CodeOverlay*`; source layers and
diagnostic codes use `overlay` and `overlay.*`. Composition, precedence, secret
resolution, and lifecycle semantics otherwise remain unchanged.
Narratio's pipeline-selection interface becomes `--overlay`, selecting
`overlays/<name>.yml` beside the root pipeline file. Remove `--profile` and old
selected-directory discovery without aliases or fallbacks. Preserve existing
base selection for omission, explicit empty selection, and `default`.
Configuration inspection, diff descriptions, internal selection names, and
newly written selection provenance use overlay terminology as well.
Maintain existing value-based digest and stage-reuse semantics; renaming
informational selection provenance is not a reason to invalidate completed
work or automatically rewrite historical manifests. Strict manifest decoding
rejects retired provenance fields; operators must deliberately migrate that
informational member on a backed-up manifest, preserving execution/checkpoint
state. Update maintained examples and operator migration guidance for the
directory, flag, and provenance changes.
This rename does not apply to downstream model profiles, Scriptorium
`profile_id`, Audita's proposal/validation profiles, or Notarius's
`config_profile`. Nor does it prohibit ordinary explicit imports from a directory
named `profiles`. Narratio overlays do not automatically select downstream
overlays or profiles. These are independently owned choices.
### Audita v2 configuration
Keep binary, timeout, explicit config-path selection, and surviving explicit
domain overrides such as transcript description, output schema, and modules.
Allow Audita's native configuration discovery when no config path is selected;
do not impose an additional requirement to supply an explicit path.
Provide optional Audita overlay, proposal-profile, and validation-profile
selectors, passed through to their corresponding CLI flags. Omission leaves
Audita's own defaults and validation-profile inheritance in control. These are
Audita selectors, not automatic translations of Narratio overlay names.
Retire the Narratio settings that mirror removed Audita endpoint/model/capacity
flags and work-directory retention. Move their operational meaning into the
operator-maintained Audita configuration and profiles. Remove stale adapter
fields, validation, invocation metadata, examples, and tests along with them.
Remove the old report toggle and both adapters' singular `llm_api_key` fields
from accepted Narratio configuration. Do not retain deprecated aliases,
automatic conversion, ignored fields, or a v1 execution path. Existing strict
configuration decoding rejects removed fields. Provide migration guidance and a
release compatibility notice rather than runtime compatibility machinery.
Preserve the distinction between omitted modules and an explicitly empty module
list: omission requests Audita's configured/default sequence; an empty list
requests no correction modules. Preserve order and repetitions rather than
sorting or deduplicating. Do not broaden this into a change to unrelated
adapters' module semantics.
Normal invocations no longer pass `--report-json`, `--work-dir-retention`, or
debug-work-directory flags as a way to obtain reports. They need no reporting
override: Audita writes its ordinary report bundle by default. The old
Narratio `report` toggle must not silently become `--reporting=false`.
The production Audita configuration must also retain reporting enabled and debug
disabled. Omitting CLI flags does not override an operator's explicit external
configuration. Document this distinction rather than silently adding forced
boolean overrides or rejecting otherwise valid Audita configuration.
## Credentials
Define secrets once under provider-oriented environment-variable names, such as
`OPENROUTER_API_KEY` and `RAKESTRAWHOME_API_KEY`, through the process environment
or Narratio's existing secrets-directory bootstrap. These names are examples,
not a fixed provider registry; application-specific names are not required.
Add `pipeline.audita.credentials` and `pipeline.notarius.credentials` as lists
of Narratio typed secret references. Each entry names the environment variable
whose captured value is forwarded under that exact name. Omission or an empty
list forwards no credentials. The same reference may appear in both adapters'
lists without duplicating its stored secret value. An illustrative configuration
fragment is:
```yaml
pipeline:
audita:
credentials:
- OPENROUTER_API_KEY
- RAKESTRAWHOME_API_KEY
notarius:
credentials:
- OPENROUTER_API_KEY
```
Resolve references through the existing configuration/secret-loading mechanism
before adapter execution, preserving its nonempty-value requirement. Forward
the captured values rather than rereading the parent environment during a run.
Retain the minimal child execution environment plus each adapter's explicit
credential list; do not forward every secret found during bootstrap.
Audita backends select these names through `credential`; the current Notarius
PromptKit profiles select them through `api_key_env`. Narratio neither inspects
downstream configuration to infer required credentials nor translates names,
registers providers, or duplicates backend configuration. Audita resolves
credentials for every surviving configured backend, even one not selected by
a profile, so its list must cover all those references.
Generalize Notarius's current single-name forwarding and replace Audita's fixed
`AUDITA_LLM_API_KEY` destination with this shared convention. Use existing shared
secret and subprocess facilities, extending narrow helpers where needed rather
than introducing a new secret-management framework. Register every forwarded
credential with captured-output redaction, including custom names and multiple
keys. Values must not enter arguments, generated configuration, manifests,
logs, or examples. As today, downstream tools remain responsible for files they
write directly; this does not add arbitrary downstream-file scanning.
## Ordinary reports and result handling
Replace the old `audita.report.json` handling with Audita's normal `errors.yml`,
`recoveries.yml`, and `report.md` bundle. With a file output, Audita writes these
beside the corrected transcript; an arbitrary report destination is not supplied
through the removed report flag.
Discover the reports at their defined run-local locations and retain them as
diagnostics using Narratio's existing manifest and diagnostic lifecycle. Keep
them distinguishable from the canonical polished transcript. Adapt report
references, publication/restore coverage, and operator inspection guidance so
the bundle is not lost or confused with an older run's JSON report. Do not
fabricate a legacy JSON report or delete historical diagnostics as part of the
migration.
Retain existing bounded, confined file handling. Replace JSON-specific report
handling with only the validation needed for consuming the v2 bundle; do not
introduce a second comprehensive report-schema validator or make incidental
Markdown wording a contract. Report locations must be discoverable without
parsing human-readable stderr. Detailed report-counter ingestion is not required
for this migration.
Audita's exit status remains authoritative. Exit zero with a valid corrected
transcript may include recoverable LLM failures, including a normalized result
with no model corrections. Preserve the reports so operators can assess that
outcome; do not add an unrequested fail-on-recovery or minimum-correction policy.
Conversely, a nonzero exit must fail polish even if a transcript file exists:
Audita can fail while writing required reports after writing the transcript.
Preserve available failure diagnostics without requiring early failures to have
produced a complete report bundle. Failed output must not become the canonical
successful result.
## Resume and semantic evidence
Update polish's semantic evidence for the new result-affecting selections and
remove retired endpoint/model fields. Include explicit config/overlay/profile
selection and preserve ordered module semantics, including omitted versus empty
selection. Version the changed evidence so incompatible older evidence is not
mistaken for a valid match when polish is considered for reuse.
Keep the existing boundary for opaque downstream configuration: Narratio does
not recursively hash Audita configuration, overlays, and profile contents.
Operators force polish when changing contents behind unchanged selectors.
Timeouts, credential values, reporting details, and other execution-only settings
must not become blanket semantic invalidation keys. Do not rerun unrelated or
unselected stages merely because the integration was upgraded.
## Migration guidance and verification
Update the canonical [configuration reference](../config.md),
[Audita integration contract](../integrations/audita.md),
[Notarius integration contract](../integrations/notarius.md),
[polish internals](../internal/stage-polish.md), maintained examples, and relevant
operations/troubleshooting guidance when implementation lands. Cover:
- Moving selected Narratio pipeline files to `overlays/`, replacing `--profile`
with `--overlay`, and distinguishing configuration overlays from model profiles.
- Updating ConfigKit API/source attribution and verifying unchanged effective
values retain existing semantic reuse behavior.
- Preparing and validating an Audita v2 configuration and its profiles before
switching the executable; copying an old v1 configuration is insufficient.
- Moving endpoint, model, capacity, and credential selections to their v2 owners.
- Replacing both singular `llm_api_key` settings with explicit `credentials`
lists, storing each provider key once, and matching downstream credential
references to those names. State clearly that old fields are removed.
- Using normal reports to inspect recoveries and failures without debug mode.
- Force/resume expectations and the treatment of existing completed sessions.
- The explicitly deferred nature of Narratio testing-profile debug support.
Verification should exercise the real adapter boundary with offline subprocess
fixtures and focused stage/configuration tests: accepted v2 arguments, omitted
defaults, ordered/empty modules, secret-safe forwarding, successful report
capture, recoverable-error success, nonzero exit with an existing transcript,
failure diagnostics, and semantic reuse/invalidation. Update existing behavioral
tests rather than retaining assertions for removed flags or exact stderr prose.
Where practical, verify invocation against the tagged Audita CLI without live
provider calls. Follow the repository's testing and documentation policies.
Credential tests must cover both adapters, multiple custom names, reuse of one
secret by both tools, no forwarding of unselected secrets, captured-value
semantics, and redaction of every forwarded value. Verify removal of obsolete
configuration fields without preserving their compatibility behavior.
## Scope boundaries
No changes to pipeline order, transcript identity, downstream artifact
definitions, or provider algorithms are intended. No automatic v1 configuration
converter, Audita-internal resume mechanism, runtime binary-version negotiation,
or live-service release gate is required. Debug-profile integration and ingestion
of Audita's debug-only execution artifacts are future work.
The Notarius change is limited to credential configuration, forwarding, and
associated diagnostics/documentation/tests; extraction and cache behavior are
unchanged. This migration breaks existing pipeline configurations using removed
fields; account for that explicitly in Narratio release versioning.

View File

@@ -0,0 +1,102 @@
---
title: Select a Notarius configuration profile through Narratio
status: proposed
consumer: rakestraw.dnd Ansible collection and its pipeline role
use_case: >-
The Ansible pipeline role deploys Narratio and Notarius as independently
configured applications. Operators select Narratio's testing profile to
develop prompts and other pipeline assets with inexpensive models while
retaining production settings in the base configuration. That selection
currently cannot select a Notarius configuration profile, leaving extraction
on its production settings.
desired_outcome: >-
Narratio's testing overlay can explicitly select Notarius's testing overlay.
Notarius owns the resulting model choices and any other testing-specific
settings. The operator selects testing once when invoking Narratio, without
editing production files or separately invoking Notarius.
rationale: >-
Both applications already use ConfigKit configuration overlays. Selecting
the downstream overlay preserves each application's configuration ownership
and supports future Notarius testing adjustments without adding a Narratio
configuration field for every downstream setting. This is explicit profile
composition, not a requirement for arbitrary recursive orchestration.
requested_interface:
field: pipeline.notarius.config_profile
value: Optional Notarius ConfigKit configuration profile name
supplied: >-
Invoke Notarius with --config-profile and the selected name when extraction
executes. The setting is available in Narratio's base configuration and
its ordinary configuration overlays.
omitted_or_null: >-
Do not supply a profile-selection flag. Preserve existing Notarius
configuration selection behavior.
downstream_semantics: >-
Retain Notarius's existing selector semantics, including the default
selector for its base configuration. Notarius resolves profiles relative
to its own configuration root, not Narratio's root.
explicit_selection: >-
Do not automatically forward Narratio's profile name. The selected
Notarius profile may have the same name or a different name.
distinction: >-
This selects a ConfigKit configuration overlay through --config-profile,
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
can itself select the desired LLM profiles using native Notarius settings.
illustrative_consumer_configuration:
note: >-
These are separate files managed by the Ansible suite, not a proposed
combined configuration document. The Notarius pipeline and LLM profile
names are illustrative; the referenced LLM profile must be available to
Notarius.
narratio_profiles_testing_yml: |
notarius:
config_profile: testing
notarius_profiles_testing_yml: |
pipelines:
dnd-session:
llm_profile: gemini-flash-latest
operator_workflow: >-
Select --profile testing on the ordinary Narratio invocation. Narratio
selects Notarius's testing configuration overlay when it runs extraction;
other Narratio testing overrides continue to apply normally.
observable_behavior:
- >-
Notarius owns overlay discovery, composition, validation, and the effective
downstream settings. A missing or invalid selected overlay must not silently
fall back to production or be reported as successful extraction.
- >-
Changing the selected Notarius configuration profile must not silently
reuse Narratio's completed extraction from the previous selection.
Existing downstream invalidation behavior applies when extraction changes.
- >-
Editing Notarius overlay contents behind an unchanged selector retains
the existing operator responsibility to force extraction when needed.
This request does not require inspecting or tracking private Notarius files.
- >-
Existing credential forwarding remains independently configurable through
notarius.llm_api_key, including in the Narratio testing overlay when the
selected Notarius settings require a different credential reference.
consumer_follow_up: >-
After Narratio releases this interface, the standalone Narratio Ansible role
can expose an optional variable and omit the field when unset. The suite role
can deploy Notarius's testing overlay and select it from Narratio's testing
overlay, then distribute the changes through the collection. These consumer
updates are separate from this upstream feature request.
scope_boundaries:
- No Notarius change is currently required; v0.7.0 supports --config-profile.
- No separate Narratio-side Notarius LLM-profile selector is requested.
- No automatic profile-name propagation or downstream configuration rewriting.
- No new credential discovery, copying, or provider-name remapping.
- This request specifies consumer behavior, not internal Narratio code changes.
references:
narratio_configuration: ../config.md
narratio_notarius_integration: ../integrations/notarius.md
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md

View File

@@ -0,0 +1,171 @@
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
Status: Implemented; release target v2.1.1
## Summary
Narratio should successfully derive a publish destination when a valid
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
This is a configuration-resolution bookkeeping defect, not a YAML formatting
restriction. The fix should preserve the existing public configuration
contract: operators may omit a derivable destination and may independently set
`required` to either `true` or `false`.
## Observed Behavior
A pipeline containing a rule equivalent to the following fails during native
configuration loading:
```yaml
publish:
outputs: [{"source": "narratio.transcript.final", "required": true}]
```
Narratio reports:
```text
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
```
The same failure is possible with ordinary block-style YAML when `required` is
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
cause of the defect.
The existing destination-derivation tests do not expose this case because
their rules also omit `required`. Narratio then supplies the default value for
`required` and records a change covering the complete `publish.outputs`
sequence. That broader record happens to cover the destination mutation as
well, masking the missing resolution record that occurs when `required` is
already explicit.
## Expected Contract
The current configuration contract remains unchanged:
- `publish.outputs[].source` is required;
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
selected built-in or configured artifact source;
- `publish.outputs[].required` defaults to `true`, but an operator may set it
explicitly to either `true` or `false`; and
- equivalent valid YAML representations have equivalent configuration
semantics.
For example, both rules below should load and validate, retain their explicit
requiredness, and receive the canonical derived destination:
```yaml
publish:
outputs:
- source: narratio.transcript.final
required: true
- source: narratio.transcript.final_markdown
required: false
```
## Root Cause
ConfigKit intentionally treats sequences as atomic when comparing values
before and after application resolution. Changing a field within one
`publish.outputs` entry is therefore observed as a change to the
`/publish/outputs` sequence.
Narratio currently derives each missing destination by recording a
normalization at the individual `publish.outputs[index].dest` path. That child
record does not cover the atomic parent-sequence change. A separate
container-level record is currently created only when Narratio also defaults
an omitted `required` value, so configurations with explicit `required` values
lose that incidental coverage.
## Recommended Fix
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
atomic-sequence contract. Whenever resolution derives one or more destinations
inside `publish.outputs`, the recorded resolution changes must cover the
`/publish/outputs` container. Destination derivation must not depend on an
unrelated `required` default being applied in the same list.
The implementation may retain more specific provenance where useful, but it
must provide valid container-level coverage for the sequence mutation. Do not
change ConfigKit's sequence comparison semantics merely to accommodate this
call site.
## Scope
- Correct destination derivation for publish rules that explicitly set
`required` and omit a derivable `dest`.
- Preserve explicit `required: true` and `required: false` values.
- Preserve canonical destination derivation for built-in transcript sources
and configured artifact sources.
- Add focused regression coverage through Narratio's real configuration load
and validation boundary.
- Prepare a backward-compatible patch release after the fix passes the normal
repository validation.
## Non-goals
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
destination derivation rules.
- Requiring operators or configuration generators to supply `dest` when it is
derivable.
- Requiring operators to omit explicit `required: true` values.
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
- Changing ConfigKit's documented atomic treatment of sequences.
- Adding storage, publication, or network integration behavior.
## Regression Coverage
Add one focused package-level behavioral test that loads and validates a
pipeline with omitted destinations and explicit requiredness. A representative
list should include both `required: true` and `required: false`, then assert
that:
- loading and validation succeed;
- each destination is derived from its source; and
- each explicit `required` value is preserved.
Using the compact flow-style representation that exposed the integration
failure is reasonable, while the existing block-style cases continue to cover
omitted `required`. The test should exercise the public configuration behavior,
not ConfigKit call choreography or Narratio's private recording mechanism.
No network service, object store, credential, or test double is required.
## Target End State
All valid publish rules with derivable sources behave consistently across
these cases:
| `dest` | `required` | Result |
| --- | --- | --- |
| omitted | omitted | destination derived; required defaults to `true` |
| omitted | `true` | destination derived; explicit `true` retained |
| omitted | `false` | destination derived; explicit `false` retained |
| explicit | either value | explicit destination and requiredness retained |
`narratio config validate`, `narratio config show`, and ordinary pipeline
loading accept the corrected cases without an unrecorded-change diagnostic.
No operator configuration migration or workaround is required.
## Release
This is a backward-compatible correction to documented configuration behavior
and should be released as Narratio `v2.1.1`. The release note should identify
the corrected explicit-required/derived-destination interaction and state that
no special upgrade action is required.
## Acceptance Criteria
- The focused regression test fails against v2.1.0 behavior and passes after
the fix.
- Publish rules with explicit `required: true` or `required: false` and omitted
derivable destinations load and validate successfully.
- Derived destinations and explicit requiredness appear correctly in the safe
effective configuration.
- Existing destination derivation, publish validation, imports, profiles, and
configuration inspection tests remain green.
- The complete repository validation and release-candidate checks pass.
- The correction is published as the immutable `v2.1.1` tag with the required
release note.

View File

@@ -0,0 +1,482 @@
# Implementation Plan: Audita v2 and Shared Provider Credentials
Status: Complete (2026-09-25).
## Scope and working instructions
Implement [the Audita roadmap](audita.md). The baseline is Narratio commit
`70f1dbf` (v2.2.1), with ConfigKit v0.4.0 and Audita v2.0.0 as the target
upstream contracts. This replaces the completed checkpoint-refactoring plan,
which remains in Git history.
Complete Stage 1 through Stage 8 in order, one bounded implementation prompt per
stage for gpt-5.6-terra. Record completion and actual verification results under
each stage. These instructions apply to the implementation sprint; the planning
pass changes documentation only. Intermediate stages are not release candidates.
Do not add compatibility code to make intermediate states look like a supported
release.
Before each stage, read `AGENTS.md`, `docs/development.md`, and all policies
under `docs/policy/`: agents, architecture, documentation, testing, and release.
Read this plan's selected design and that stage's preparation. Use Go 1.26 and graph-first code
discovery; inspect current files when graph results are incomplete. Preserve
unrelated changes. Test at behavioral owners rather than replicating every case
across configuration, adapter, stage, and command tests.
The user approved a clean cutover: no Audita v1 runner, deprecated aliases,
compatibility shims, generated configuration conversion, or silently ignored
old settings. Do not broaden that decision into a purge of unrelated legacy
code or historical session data.
## Selected design
### ConfigKit and Narratio overlay terminology
Upgrade ConfigKit from v0.3.1 to v0.4.0 before adapter work. Rename its
`Options.Profile`, `LayerProfile`, and any `CodeProfile*` uses to their overlay
equivalents. Selected pipeline files move from `profiles/<name>.yml` to
`overlays/<name>.yml`; no fallback directory or alias is permitted.
Rename Narratio's pipeline-selection flag to `--overlay` everywhere it is
accepted, including convenience commands and configuration inspection. Rename
application-owned selection types/helpers, help, errors, and current inspection
labels consistently. `config diff` still takes two positional selections,
now described as overlays. Preserve omission, explicit empty, and `default`
as base selection; do not add a persistent default selector or new precedence.
Rename newly written informational provenance to overlay terminology, including
`selected_overlay` in manifest configuration provenance. Do not add a dual-field
reader, automatically rewrite historical manifests, or introduce a manifest-version
rejection solely for this informational rename. Existing strict typed decoding
rejects retired provenance fields. Document backed-up operator migration of
that informational member without changing execution/checkpoint state; it does
not become resume authority. Preserve
semantic fingerprints and effective-value digest rules when values are unchanged.
Only Narratio pipeline configuration selection is renamed. Keep model/profile
fields such as Scriptorium `profile_id`, Audita `proposal_profile` and
`validation_profile`, and Notarius `config_profile` unchanged. Explicit imports
from directories named `profiles` remain legal; only selected pipeline overlay
discovery changes. Do not globally replace the word “profile.”
### Configuration and credentials
- Both `pipeline.audita.credentials` and `pipeline.notarius.credentials` are
lists of ConfigKit secret references. Use `[]configkit.Secret`; each element
is an actual reference rather than a nullable pointer. Existing ConfigKit
validation owns name/value resolution. Omitted, null, or empty lists mean no
additional credential forwarding.
- Define each key once through existing environment/secrets-directory bootstrap.
Forward it under its original name to each adapter whose list includes it.
No alias mappings, provider registry, downstream config inspection, extra
bootstrap, or inheritance of the entire environment.
- Repeated names need no new rejection rule: forwarding the same captured
reference once suffices. Arbitrary valid variable names remain supported;
provider names in the roadmap are examples, not an allowlist.
- Add a narrow shared helper in `internal/adapters/subprocess` converting resolved
references to `EnvOverrides` and `SensitiveEnvNames`. It checks the existing
resolved/nonempty requirement for direct adapter callers but never resolves
secrets or rereads environment values. Errors expose no values. Both adapters
use it; the ConfigKit upgrade is owned by Stage 1, not this helper. No new
credential service is needed.
- Configuration projections and diagnostic metadata contain reference names
only. Register all forwarded names for existing captured-stream redaction.
Downstream tools remain responsible for files they write themselves.
- Remove both singular `llm_api_key` fields. For Audita also remove `base_url`,
`model`, `validation_model`, `total_llm_concurrency`,
`proposal_llm_concurrency`, `validation_llm_concurrency`,
`work_dir_retention`, and `report`, including defaults and projections.
- Retain Audita `binary`, `timeout`, `config_path`, `transcript_description`,
`output_schema`, and `modules`. Add optional string selectors `overlay`,
`proposal_profile`, and `validation_profile`: empty/unset means no CLI
override. Audita owns downstream file/profile validation and native config
discovery. Narratio overlay names are not implicitly Audita overlay names.
- Preserve nil versus non-nil empty `modules` through loading, projection,
cloning, composition, adapter defaults, and request overrides. Nil omits the
flag; empty passes `--modules` with an empty argument; nonempty joins in order
with repetitions intact. A nil request override uses the configured selection.
Avoid clones that collapse empty slices to nil. YAML null follows unset
behavior. Other adapters' module semantics do not change.
### Invocation, reports, and failure
Use `process INPUT --glossary PATH --output PATH` plus selected surviving
overrides. Production passes no reporting or debug switches, including
`--debug`, `--reporting=false`, `--report-dir`, or `--work-dir`.
Remove required `WorkDir` and legacy `ReportPath` request/result plumbing.
Document production external-config defaults without forcibly overriding or
rejecting deliberate external reporting/debug choices.
Replace Audita's long positional config-values constructor with typed runner
configuration. Keep timeout parsing in composition and subprocess cancellation
through the existing runner. No version probe or automatic config-validation
subprocess is required.
The corrected transcript remains the only canonical polish output, with existing
identity, path, validation, and bounded file handling. Discover exactly
`errors.yml`, `recoveries.yml`, and `report.md` beside the requested output.
Represent available paths in a small typed report bundle in `PolishResult`.
Fixed names belong to the adapter; do not parse stderr or recursively scan.
Reports are diagnostics, not a new correctness gate:
- Bound reads of present reports using the existing named 16 MiB report limit;
retain confined regular-file handling and errors identifying Audita, the file,
and limit. No new smaller limits or report schema/status/count validation is
necessary when Narratio only preserves bytes; Markdown needs no parser.
- Missing files are not independently fatal: external configuration can disable
reports and early failures may produce none. Default-reporting success tests
must nevertheless demonstrate capture of all three files.
- On process failure, preserve available diagnostics and the original error;
report-inspection errors must not hide it. Never promote an output merely
because it exists following a nonzero exit.
- Exit zero with a valid transcript may include recoverable errors or no model
corrections. Accept it without a fail-on-recovery or minimum-correction rule.
Keep reports next to the run-local transcript, not in a second canonical
destination. Register present report paths in the existing `StageResult.Logs`
diagnostic collection alongside stdout/stderr, with safe bundle-path metadata.
Reuse current manifest-driven diagnostic publication rather than adding a
manifest schema or new publish source IDs. Physical placement does not define
canonical artifact identity. Existing diagnostic retention/cleanup rules apply.
On failure, retain run-local files and expose the output/report directory in
contextual errors. Do not add runner-wide failed-result persistence merely to
register those files. Do not fabricate legacy JSON reports or delete historical
reports. Normal successful stage-record replacement removes obsolete current
output references.
Generated invocation YAML becomes `audita.generated.v2`, recording actual
arguments, selections, paths, and credential names, never values. This remains
a diagnostic invocation record, not an Audita configuration file.
### Resume boundary
Bump only polish semantic-config version from 1 to 2. Record config path,
overlay/proposal/validation selectors, ordered modules with presence preserved,
transcript description, output schema, and existing output identity. Remove old
endpoint/model fields. Existing runner comparison reconsiders obsolete evidence
when polish is selected; do not synthesize equivalent fingerprints or expand
normal dependency/selection invalidation.
Credentials, binary path, timeout, and diagnostics are not semantic keys. Do not
hash downstream config/profile contents; operators force polish after edits
behind unchanged selections. Notarius credential migration does not change
extraction fingerprints, references, force, resume, or chunk-cache behavior.
## Stage 1: Adopt ConfigKit v0.4.0 and pipeline overlays
Status: Complete (2026-09-25).
Verification: `go test ./internal/config ./internal/app ./internal/manifest`,
`go test ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check` passed.
Preparation: ConfigKit v0.4.0 release notes and selected-overlay migration guide
in `../ConfigKit/docs/`; Narratio `internal/config/pipeline_configkit.go`,
`artifact_catalog.go`, `load.go`, `effective_digest.go`; CLI flag parsing and
config commands in `internal/app/`; configuration provenance in
`internal/manifest/manifest.go`; current config/CLI docs and pipeline examples.
1. Update `go.mod`/`go.sum` to ConfigKit v0.4.0 using normal module tooling;
do not leave a local replacement or change unrelated dependencies.
Replace removed ConfigKit APIs in loading, ownership, and artifact catalog
inspection. Account for source layer `overlay` and diagnostic prefix
`overlay.*` wherever they are consumed or asserted.
2. Rename Narratio's pipeline selector to `--overlay` across all commands and
aliases, load options, pair loading, inspection, provenance, and tests.
Preserve explicit-presence handling through convenience commands and existing
base/default selection. Remove `--profile` rather than retaining an alias.
Apply the selected design's informational manifest-provenance treatment.
3. Move maintained selected pipeline files and generated fixture paths to
`overlays/`. Update current canonical documentation, policy terminology,
examples, and user-visible migration guidance with the directory/flag break.
Do not rewrite historical release notes, rename downstream model profiles,
or prohibit ordinary explicit imports from a directory named `profiles`.
4. Adapt existing config/app tests to prove base and named-overlay selection,
imported defaults plus overrides, secret replacement/removal, source attribution
through character-family expansion, config diff, and convenient-command
forwarding. Add focused checks that the retired flag is rejected and the old
directory is not a discovery fallback. Do not freeze incidental error wording.
5. Verify unchanged effective values retain digest and stage-reuse behavior;
only diagnostic source labels/paths and selected provenance terminology change.
Test current provenance round trips and strict rejection of unknown/retired
provenance fields. Document deliberate operator migration; do not add a
compatibility decoder, semantic invalidation rule, or schema-version gate.
6. Run focused and full tests before proceeding so failures from the dependency
and terminology change are separated from adapter migrations. No Audita or
Notarius credential/CLI behavior changes in this stage.
Verify: `go test ./internal/config ./internal/app ./internal/manifest`,
`go test ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check`.
Done when Narratio uses ConfigKit v0.4.0, exposes pipeline overlays consistently,
has no old selector/directory fallback, and preserves configuration semantics and
downstream model-profile concepts.
## Stage 2: Add shared resolved-credential forwarding
Status: Complete (2026-09-25).
Verification: `go test ./internal/adapters/subprocess`,
`go test -race ./internal/adapters/subprocess`, and `git diff --check` passed.
Preparation: `internal/adapters/subprocess/run.go`, its environment/redaction
helpers and tests; `internal/adapters/notarius/subprocess.go`;
`internal/config/credential_projection.go`; the pinned ConfigKit secret API.
1. Add the narrow helper described above. Return an independent environment map
and selected sensitive names. No logging/serialization of values and no
changes to minimal child environment inheritance.
2. Test multiple custom names, empty lists, repeated names, unresolved references,
and captured values after the parent environment changes. Reuse existing
split/overlapping-stream redaction coverage rather than repeating its matrix.
3. Leave production callers unchanged until their migrations.
Verify: `go test ./internal/adapters/subprocess` and
`go test -race ./internal/adapters/subprocess`.
Done when both adapters can reuse one small conversion mechanism without adding
a resolver, provider policy, or ambient-secret inheritance.
## Stage 3: Migrate Notarius to credential lists
Status: Complete (2026-09-25).
Verification: `go test ./internal/config ./internal/adapters/notarius ./internal/stage ./internal/app`,
`go test -race ./internal/adapters/notarius`, and `git diff --check` passed.
Preparation: `internal/config/config.go`, `credential_projection.go`, loading/
inspection tests; `internal/adapters/notarius/runner.go`, `subprocess.go`;
`internal/stage/extract.go` and its request construction;
`docs/integrations/notarius.md` and secret handling in `docs/config.md`.
1. Replace the singular field with the typed list across configuration,
projection, request types, stage wiring, and adapter execution. Use Stage 2's
helper and register all selected names for redaction.
2. Update name-only metadata and config-show/diff/effective projections. Test
real ConfigKit loading, overlay list replacement, and empty-list clearing.
Preserve bootstrap precedence and existing load-time reference resolution,
including references outside the selected stage range.
3. Migrate maintained examples, fixtures, and current Notarius configuration/
integration documentation for this field. Ordinary strict decoding rejects
the old field; do not add a compatibility parser.
4. Cover multi-key forwarding, unselected-key exclusion, empty selection,
captured-value behavior, and custom-name redaction on success/failure.
Keep extraction cache/reference/output tests as regression protection,
not occasions to redesign extraction.
Verify: `go test ./internal/config ./internal/adapters/notarius ./internal/stage ./internal/app`
and `go test -race ./internal/adapters/notarius`.
Done when Notarius uses credential lists end to end and no active singular
credential path remains.
## Stage 4: Cut Audita configuration and adapter over to v2
Status: Complete (2026-09-25).
Verification: `go test ./internal/config ./internal/adapters/audita ./internal/stage ./internal/app`,
`go test -race ./internal/adapters/audita`, `go test ./...`, and
`git diff --check` passed.
Preparation: Audita v2.0.0 `docs/migrations/v2.md`, `docs/cli.md`,
`docs/integrations/subprocess.md`, and `docs/integrations/output-files.md` in
`../audita/`; Narratio `internal/adapters/audita/`,
`internal/app/runner.go` (`buildDefaultAuditaRunner`), and config types,
defaults, validation, and credential projection.
1. Replace the Audita configuration surface as selected above, updating defaults,
validation, safe projection, examples, and config fixtures. Prove nil/empty
modules survive real configuration loading and overlay overrides.
2. Migrate runner construction, selectors, and credential forwarding. Remove
retired flags, fixed credential alias, required work-directory plumbing, and
old JSON-report validation. Add the typed report bundle.
3. Implement adjacent report discovery, bounded reads, failure results, and
generated invocation v2. Share metadata construction between success/failure
instead of maintaining duplicate field maps.
4. Rewrite the Audita subprocess fixture to model v2 argument acceptance and
output placement. Cover omitted/selected selectors, ordered/repeated/empty
modules, no report/debug switches, malformed/missing transcript, ordinary
reports, externally disabled reporting, recoverable-error success, and
nonzero exit after transcript creation. Cover multiple custom credential names
and redaction; preserve timeout/cancellation coverage at its existing owner.
5. Make mechanical polish and app-fixture adjustments needed to compile: remove
obsolete request/metadata fields and old report materialization. Stage 5 owns
complete diagnostic integration. Remove retired fingerprint members as needed
to compile; Stage 6 completes the replacement semantic contract. These are
bounded call-site changes, not permission to introduce transitional shims.
Verify: `go test ./internal/config ./internal/adapters/audita ./internal/stage ./internal/app`
and `go test -race ./internal/adapters/audita`.
Done when configuration composes a v2-only adapter and subprocess tests prove its
credential/transcript/report boundary. Stages 5 and 6 complete diagnostic and
semantic integration; all stages are required before release.
## Stage 5: Integrate polish reports with diagnostic lifecycle
Status: Complete (2026-09-25).
Verification: `go test ./internal/stage ./internal/app ./internal/manifest ./internal/artifacts`,
`go test -race ./internal/stage ./internal/app`, and `git diff --check` passed.
Preparation: `internal/stage/polish.go`, `polish_test.go`, `run_local.go`,
`stage.go`, `publish_sources.go`; result application in `internal/app/runner.go`;
publish/restore diagnostic tests; `docs/internal/stage-polish.md`.
1. Finish stage integration: materialize only the canonical polished transcript;
register available adjacent reports as diagnostics plus safe metadata and
existing logs/generated invocation references. Remove old `audita_report`
production and stale stage metadata.
2. Use existing authoritative-path checks for the requested transcript and fixed
adjacent report paths. Do not accept arbitrary fake-returned paths, scan
directories, or copy reports to a second destination. Preserve workspace
handling/modes.
3. On execution failure, preserve canonical transcript authority and run-local
diagnostics, with actionable directory/log context and original error identity.
Test failure after output creation.
4. Adapt the fake and app helper. With real manifest/store collaborators, prove
successful promotion, report references, absence of report canonical outputs,
and current-run diagnostic publication. Preserve restore's existing scope of
declared committed transcript/artifact objects: run-archive diagnostics do not
become automatically restored canonical objects. Test that report references
do not create new required canonical restore targets. Document remote diagnostic
inspection separately; no new diagnostic-restore feature is authorized.
5. Leave historical files untouched. Update tests expecting a second canonical
polish output without introducing a manifest-format migration.
Verify: `go test ./internal/stage ./internal/app ./internal/manifest ./internal/artifacts`
and `go test -race ./internal/stage ./internal/app`.
Done when reports use existing diagnostic lifecycle and failed execution cannot
promote an incidental transcript.
## Stage 6: Complete semantic evidence and resume behavior
Status: Complete (2026-09-25).
Verification: `go test ./internal/stage ./internal/app`,
`go test -race ./internal/stage ./internal/app`, and `git diff --check` passed.
Preparation: `internal/stage/semantic_contracts_refinement.go`, semantic
fingerprint tests, app runner semantic comparison tests, and extraction semantic
configuration tests (locate their current owners through graph discovery).
1. Implement polish evidence version 2 and the selected payload. Remove sorting;
preserve nil/empty distinction in payload encoding and copies.
2. Test changes from module order/repetition/presence and external selectors.
Credentials and execution-only settings do not change evidence. Avoid
assertions on incidental digest literals.
3. Exercise old-version evidence through runner resume behavior: selected polish
is reconsidered and normal dependent invalidation applies. Downstream-only
bounded execution does not unexpectedly invoke polish. Unchanged new evidence
reuses valid completed output.
4. Confirm credential-list changes leave extraction semantic/cache behavior
unchanged. No downstream file-content scanning or eager fingerprint migration.
Verify: `go test ./internal/stage ./internal/app` and
`go test -race ./internal/stage ./internal/app`.
Done when selections affect reuse accurately without blanket credential/config
invalidation or changes to unselected-stage execution.
## Stage 7: Complete operator migration documentation and examples
Status: Complete (2026-09-25).
Verification: `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check` passed.
Preparation: documentation policy; `docs/config.md`, `docs/operations.md`,
`docs/troubleshooting.md`, both integration guides,
`docs/internal/adapters.md`, `docs/internal/stage-polish.md`, and pipeline examples.
1. Complete current-behavior documentation as implementation lands. Configuration
owns fields/defaults; integrations own external contracts; operations owns
placement and migration; internal docs explain ownership and link onward.
2. Provide an old-to-new migration table: endpoints/capacity move to Audita
backends, models to profiles, role selection to selectors, and credentials to
provider-named lists. Remove report/retention settings. Explain Audita's
all-configured-backend credential resolution and Notarius's profile lookup.
3. Describe preparing and validating Audita v2 config offline with its documented
command before switching executables. Link upstream examples rather than
building a downstream configuration generator. Explain ordinary reports,
recoverable-error success, and force after edits behind unchanged selectors.
4. Production examples omit modules when upstream defaults are intended;
`modules: []` deliberately disables correction modules. Production external
Audita config keeps reports enabled/debug disabled, without Narratio flag
overrides. Testing-profile debug integration remains deferred.
5. Document the configuration-breaking clean cutover for release-version
consideration. Do not edit historical release records, tag a release, or
remove unrelated roadmaps.
Verify: `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check`.
Done when operators can migrate using valid examples without obsolete fields,
implicit credential aliases, or retired CLI/report expectations.
## Stage 8: Verify assembled behavior and audit scope
Status: Complete (2026-09-25).
Verification: the offline app-level shared/Audita-only credential scenario
passed with config bootstrap, both subprocess fixtures, safe inspection/logging,
and ordinary polish reports. The tagged Audita v2.0.0 CLI validated its minimal
configuration and produced a transcript plus all three reports with an empty
module sequence and no provider request. Active-reference and scope audits found
no removed production fields or compatibility path; remaining old names occur in
migration guidance, explicit-rejection checks, or historical fixtures. `go test
./...`, `go test -race ./...`, `go vet ./...`, `go build ./...`, `go test
./internal/doccheck`, `go test ./internal/config -run
'^TestExamplesLoadAndValidate$'`, and `git diff --check` passed.
Preparation: cumulative diff, this plan, feature roadmap, app configuration/
subprocess integration tests, and all policies.
1. Adapt or add one representative offline app-level scenario showing one
provider-named secret source supplying both adapters and another key selected
only for Audita. Use real config/bootstrap/composition with subprocess fixtures.
Prove safe inspection/logging and successful polish reports. Leave exhaustive
flag/redaction cases at lower layers.
2. Where the tagged Audita checkout is available, smoke-test CLI acceptance and
ordinary transcript/reports offline with an explicit minimal v2 config,
unauthenticated backend/profile fixtures, and an empty module sequence.
No provider requests or production credentials. This is optional local evidence,
not a default test dependent on sibling checkouts or installed tools.
3. Audit active references to removed fields/flags/aliases; historical release
notes and completed roadmaps are not compatibility code. Review tests
collectively for duplication while retaining consequential failure coverage.
4. Confirm no provider-name restrictions, duplicate-name rejection, downstream
content hashing, fail-on-recovery rule, mandatory explicit config path, debug
requirement, live-service gate, pipeline-order change, or Notarius cache change
slipped in. Do not require missing reports when disabled externally. Surface
additional policy choices rather than encoding them silently in tests.
5. Run every gate below and record actual results. Mark complete only after all
stages and gates pass. No release, tag, push, or unrelated cleanup in this plan.
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go test ./internal/doccheck
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
git diff --check
```
Done when the assembled v2 integration and shared credentials meet the roadmap,
verification passes, and no compatibility machinery remains within scope.
## Open Questions
None requiring a product decision before implementation. The user settled the
clean cutover and shared provider-named lists. Reports use existing diagnostics
without new content-based success criteria; debug-profile work remains deferred.
Release version selection belongs to the later release procedure.

View File

@@ -0,0 +1,151 @@
# Notarius Configuration Profile Selection
Status: Implemented.
## Goal and Authority
Implement the consumer behavior in the
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
configuration overlay explicitly select a Notarius configuration overlay.
Operators can select testing once in Narratio while each application retains
ownership of its own model choices and other settings.
This roadmap describes the target state and required change areas, not an
implementation sequence; the [implementation plan](implementation.md) owns
the ordered work and implementation decisions. The feature request is the scope
authority; the mechanisms suggested below do not establish additional
user-facing contracts.
## Underlying Issue
Narratio can select its own testing profile, but extraction currently invokes
Notarius without a configuration-profile selector. Consequently, a Narratio
testing invocation cannot explicitly select the independently deployed Notarius
testing overlay through pipeline configuration.
Adding individual downstream model or tuning fields would duplicate Notarius's
configuration responsibilities. Passing its existing profile selector provides
the requested flexibility without that duplication.
## Targeted End State
### Explicit Configuration Composition
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
name, usable in Narratio's base configuration and ordinary overlays. For
example, Narratio's `profiles/testing.yml` may contain:
```yaml
notarius:
config_profile: testing
```
Selecting Narratio's `--profile testing` then selects that explicit downstream
profile whenever extraction executes. The two profile names need not match.
Narratio does not automatically forward its own profile name.
When supplied, the selection is passed through Notarius's `--config-profile`
argument. Omitted or null values add no profile-selection flag and preserve
existing downstream selection behavior. Notarius's selector semantics remain
authoritative, including empty and `default` selections for the composed base
configuration. Narratio does not introduce a stricter profile-name policy.
This field selects a ConfigKit configuration overlay, not a PromptKit profile
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
selection. Notarius resolves overlays relative to its own configuration root;
Narratio neither resolves those files against its own root nor rewrites them.
### Execution, Reuse, and Failure
A changed downstream profile selection must not silently reuse Narratio's
completed extraction from the previous selection. Existing dependent-stage
invalidation applies when extraction changes; the feature does not change the
pipeline order or transcript-generation behavior.
Notarius continues to own downstream overlay discovery, composition, validation,
and cache compatibility. A missing or invalid selected overlay must fail
extraction through the existing subprocess error path, without silently falling
back to production or reporting successful extraction.
Existing resume and chunk-cache controls, including force precedence, remain
in place. Profile selection does not itself mean forced regeneration or cache
deletion. Narratio's decision to execute extraction and Notarius's decision to
reuse compatible work remain separate responsibilities.
Changes to private Notarius configuration behind an unchanged selector retain
the existing operator responsibility to force extraction when needed. There is
no new downstream file inspection or content-tracking requirement.
Credential forwarding remains independently configured through
`notarius.llm_api_key`. A Narratio testing overlay may select a different
credential reference alongside the downstream profile; no new credential
discovery or provider-name mapping is introduced.
## Required Change Areas
### Configuration and Inspection
Extend the typed Notarius configuration and its existing safe effective-YAML
projection to carry the optional field. Preserve the ordinary configuration
composition behavior for base values, overlay overrides, and null values.
Configuration inspection must reflect the selected field without exposing
credential values. No replacement configuration loader or downstream preflight
loader is needed.
### Stage and Adapter Boundary
Carry the selection from resolved configuration through the extraction request
to the subprocess adapter. Construct the optional CLI flag using the existing
argument-vector mechanism, alongside the current configuration path, references,
credential forwarding, and cache controls. Retain normal Notarius error handling.
### Extraction Reuse Evidence
Account for the downstream selection in the existing extraction reuse evidence.
The current semantic-configuration and invocation fingerprints are the relevant
change points. Keep their treatment of selection consistent so that a profile
change cannot bypass the required rerun and downstream invalidation.
This roadmap does not require a new manifest schema, separate profile-indexed
workspaces, historical per-profile result retention, or an upgrade-wide forced
rerun.
### Verification and Documentation
Use focused behavioral coverage for configuration composition and inspection,
explicit argument forwarding and omission, changed-selection reuse decisions,
and ordinary subprocess failure handling. Reuse existing test coverage where
it already protects unchanged credentials, force behavior, and invalidation;
do not duplicate Notarius's complete profile-validation or cache test suites.
When implemented, update the canonical
[configuration reference](../config.md),
[Notarius integration contract](../integrations/notarius.md),
[extract internals](../internal/stage-extract.md), and relevant maintained
examples. The development workflow in [Operations](../operations.md) should
explain explicit profile composition and the existing need to force extraction
after private downstream edits. Clarify that profile selection does not create
an isolated testing workspace or separate current extraction state.
## Compatibility and Scope Boundaries
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
required. Document the capability needed when the option is used, without adding
an unconditional runtime version gate for existing unconfigured deployments.
This feature adds no Narratio CLI selector, automatic profile propagation,
Notarius LLM-profile selector, downstream configuration rewriting, new
credential mechanism, or generalized recursive orchestration. Existing output
and receipt contracts remain unchanged.
The consumer's Ansible role and suite updates are follow-up work outside this
repository. They can deploy downstream overlays and populate this field after
the interface is released.
## Scope Review
No stronger requirements or new invariants are proposed beyond the feature
request and existing repository contracts. In particular, minimizing one-time
upgrade invalidation and avoiding reruns between equivalent base selectors are
implementation considerations, not additional acceptance guarantees imposed
by this roadmap. No user decisions are currently required before planning.

227
docs/roadmap/remediation.md Normal file
View File

@@ -0,0 +1,227 @@
# Audita v2 and ConfigKit Migration Review Remediation
Status: Complete.
Reviewed `83eaa96` through `fae18bf` against the approved clean-cutover decisions,
[feature roadmap](audita.md), and [implementation plan](implementation.md).
Review date: 2026-09-25. No production changes were made. The previous completed
transcription review is retained below as historical context.
## 1. Preserve omitted modules in direct YAML serialization
Priority: P2. Status: Complete.
The safe YAML projection now omits nil modules while retaining explicit empty
and ordered/repeated selections. A round-trip regression failed before the fix
and passes afterward. Config package tests, including credential-safety coverage,
and its race tests pass.
Location: `internal/config/credential_projection.go:25` (`AuditaConfig.MarshalYAML`),
especially the plain slice field at line 30 and assignment at line 40.
The direct safe YAML projection passes a nil module slice to yaml.v3, which
serializes it as `modules: []`. Loading that output produces a non-nil empty
slice. Under the new Audita contract these are different instructions: nil
delegates to Audita's configuration/default sequence; empty passes an explicit
empty module argument and disables correction modules. Serialization therefore
silently changes behavior for a configuration that is subsequently reloaded.
Evidence: a temporary package test marshaled
`AuditaConfig{Binary: "audita", Timeout: "3h"}`, unmarshaled it into another
`AuditaConfig`, and required `Modules == nil`. It failed with output containing
`modules: []`. The test was removed after reproduction. This concerns direct
`yaml.Marshal` of the typed configuration; ordinary CLI `config show` uses
`MarshalEffectivePipeline` through a separate JSON/value-tree projection and
does not use this faulty path.
Required remediation:
- Make the direct safe YAML projection preserve nil as null or omission and
explicit empty as `[]`. Do not simply add `omitempty` to the slice: that would
instead erase the explicit empty selection.
- Keep credential serialization name-only and preserve module order/repetitions.
- Add a small round-trip regression covering nil, explicit empty, and an ordered
repeated selection; retain effective-output inspection coverage without
duplicating its whole test matrix.
## 2. Remove the unapproved manifest compatibility decoder
Priority: P2. Status: Complete.
Removed the permissive custom decoder; ordinary strict typed decoding now
applies to provenance as well. Replaced historical-acceptance coverage with
retired/unknown-field rejection tests, all of which failed before removal.
Current provenance round-trip coverage remains intact. Corrected the roadmap,
plan, internal contract, and operator guidance for narrow backed-up migration
without discarding execution/checkpoint state or changing resume authority.
Location: `internal/manifest/manifest.go:79-89`,
`internal/manifest/store_test.go:78`, and `docs/internal/manifest.md`'s decoding
contract. Also correct the conflicting migration language in the plan/roadmap.
The new `EffectiveConfigProvenance.UnmarshalJSON` bypasses the manifest store's
`DisallowUnknownFields` by using permissive `json.Unmarshal` for the entire
provenance object. It accepts retired `selected_profile` data and silently
discards it. It also silently accepts unknown current provenance fields,
including unknown nested selection fields, rather than limiting behavior to
the renamed label.
The user expressly selected a clean cutover without backward-compatibility
shims. This adds precisely such a shim and a new historical-input acceptance
guarantee. The regression test explicitly requires the old selection to load,
and the internal guide now declares permissive informational-field decoding.
The plan contributed to this divergence: it said existing generic decoding
would handle historical optional provenance and required old manifests to load.
That assumption was false; the baseline store already strictly decoded these
fields. Repetition in the plan, tests, and documentation does not authorize
overriding the user's clean-cutover decision.
Required remediation:
- Remove the custom compatibility unmarshaler and retain ordinary strict typed
decoding for the new `selected_overlay` representation. Do not add dual fields,
special retired-field handling, automatic manifest rewrites, or a format-version
bump just for this rename.
- Replace the historical-acceptance test with coverage for current provenance
round trips and unknown/retired provenance rejection through the real store.
- Correct the plan and current documentation to acknowledge the upgrade effect:
manifests with retired provenance require deliberate operator migration before
loading. Document narrow removal/renaming of retired informational provenance
on a backed-up manifest, preserving execution/checkpoint state; do not advise
deleting all session progress or rerunning everything.
- Keep provenance informational: this correction does not authorize adding it
to semantic resume keys or invalidating work based on overlay identity alone.
If transparent historical-manifest acceptance is desired instead, obtain an
explicit exception to the clean-cutover decision before retaining a compatibility
reader. Do not treat that exception as already granted.
## Review verification
The following records the original review, before remediation.
- Existing `go test ./...` passed.
- Race tests passed for subprocess, Audita, Notarius, config, stage, app, and
manifest packages after removing the temporary reproduction.
- `go vet ./...` passed.
- The temporary direct-YAML regression failed as described above.
- Source inspection and the new historical-provenance test establish finding 2;
`LocalStore.LoadReader` still uses strict decoding outside the custom hook.
The primary architecture is otherwise in place: ConfigKit v0.4.0 overlays,
shared named credential forwarding/redaction, a v2-only Audita subprocess
adapter, ordinary reports as diagnostics, and versioned ordered-module polish
evidence. No live model/provider runs were performed during this review.
## Migration remediation verification
Completed on 2026-09-25. Both new regressions failed before their respective
fixes. After remediation, `go test ./...`, `go test -race ./...`,
`go vet ./...`, `go build ./...`, documentation checks, maintained configuration
example checks, and `git diff --check` passed. No live provider runs were needed.
## Historical completed transcription review
Status: Complete.
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
upload-replay roadmap and the original user request. No production changes were
made during review. The two failures below were reproduced with temporary tests
using the existing stage fixtures; those temporary tests were removed afterward.
### 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
Priority: P2. Status: Complete.
Implemented read-only aggregate identity/evidence checks before resume skips.
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
repair without HTTP requests, downstream invalidation, and exclusion of removed
speakers from merge without deleting their files. Equivalent path forms and
output ordering remain reusable. Focused app/stage tests and documentation
checks pass.
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
consumed there.
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
only establishes that the remaining speakers need no transcription; it does
not establish that the stored speaker collection and aggregate outputs match
the current input set. Removing a speaker while the remaining inputs are
unchanged produces an empty work list, so a succeeded record can be skipped
without pruning the removed speaker. Its old aggregate output can then remain
available to merge. Likewise, missing/inconsistent aggregate output references
are not repaired when all individual checkpoints remain valid. A prepare rerun
may already stale transcribe in some workflows, but the resume validator cannot
rely on that to satisfy its own current-input/output checks.
Reproduction: create complete alice/bob checkpoint records and aggregate
outputs with the existing fixture, mark the stage succeeded, remove bob from
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
despite still retaining bob in the stored collection and aggregate outputs.
Required change:
- Distinguish no HTTP work from no reconciliation work. Validate that the
committed speaker set and aggregate transcript references represent the
reconciled current set, including relevant identity/evidence, before skipping.
- When only collection/output repair is needed, return non-resumable and let
normal execution checkpoint the pruned collection and rebuild aggregate
outputs without retranscribing reusable speakers. Preserve existing dependent
invalidation; do not force otherwise-current speaker jobs.
- Keep resume inspection read-only. Do not fix this by deleting incidental
transcript files or changing stage selection rules.
Verification: add removed-only input coverage (not just removal combined with
an added speaker), plus incomplete/inconsistent aggregate output coverage.
Exercise the runner skip path and assert zero additional WhisperX requests,
correct rebuilt outputs, and no removed speaker reaching merge.
### 2. Remove the Unrequested Positive Audio-Size Requirement
Priority: P2. Status: Complete.
Audio evidence now permits zero size while still rejecting negative values.
Regression tests verify zero-size manifest round trips, successful service
results being checkpointed and reused, and service rejection being propagated
without creating a completion record. Nonempty valid-JSON output validation is
unchanged. Both new success regressions failed before the fix.
Location: `internal/manifest/transcribe_state.go:95`.
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
a regular `.flac` file and delegated transcription acceptance to WhisperX;
neither the user request nor the roadmap introduced minimum audio size or a new
audio-content validation policy. Zero-byte content still has a valid checksum
and can be represented by the checkpoint mechanism. The current check runs
after a successful response and output materialization, causing the entire
stage to fail instead of retaining the completed work.
Reproduction: prepare one speaker, replace its prepared audio with an empty
regular file, and use the existing successful fake WhisperX client. Stage
execution fails at checkpoint validation with `audio_size must be positive`.
This demonstrates the extra Narratio restriction; it does not assert that a
particular production WhisperX deployment accepts empty audio.
Required change:
- Permit zero audio size and reject negative sizes. Keep byte hashing and
size equality checks. Let the downstream service decide whether the input
can be transcribed, as before.
- Do not remove the nonempty valid-JSON output requirement, which has a separate
existing justification. Do not add a new early empty-audio rejection instead.
- Update any new tests/documentation that promote strictly positive input size
to a contract.
Verification: cover a zero-size record round-trip and successful adapter result
being checkpointed/reused for that input; retain negative-size rejection and
ordinary downstream-failure tests.
### Verification After Remediation
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
changed-package race tests, `go test ./...`, `go test -race ./...`,
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check` all passed. Existing replay and cancellation regression
tests remain intact; the HTTP adapter required no changes.

View File

@@ -0,0 +1,183 @@
# Resumable Speaker Transcription and Replayable Uploads
Status: Implemented.
## Goal and User Intent
Preserve completed speaker transcriptions when another speaker fails, so an
ordinary run or resume does not repeat expensive successful work. Separately,
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
recoverable connection or stream failure.
The motivating incident exhausted four transcription attempts for one speaker
after three other speakers had completed. The final error reported an HTTP/2
peer `PROTOCOL_ERROR` and an inability to replay the request body without
`Request.GetBody`.
These are related reliability improvements, not a promise to eliminate all
service failures. This document records the target state and scope. The
completed implementation sequence is retained in Git history; current behavior
is documented in the canonical guides linked below.
## Pre-implementation Limitations
The [transcribe stage](../internal/stage-transcribe.md) writes successful
responses into the invocation's run directory. It installs canonical transcripts
and returns its per-file metadata only after every speaker succeeds. A failure
therefore leaves no durable per-speaker progress that ordinary resume can use,
even though some completed JSON files may remain on disk.
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
the application level, reopening audio for each attempt. Its multipart request
body is streamed through a pipe and has no replay callback. Transport-level
recovery within an attempt cannot reconstruct a consumed body. Providing that
capability addresses the replay limitation, not the underlying peer protocol
error; persistent failures can still require service or proxy investigation.
## Targeted End State
### Durable Per-Speaker Completion
Each successfully transcribed speaker file is validated, saved, and recorded
durably before that work is treated as reusable. The session manifest remains
the progress authority. Follow the existing analyze-stage checkpoint pattern
without turning speakers into pipeline stages or introducing a general task
engine.
Ordinary execution reconciles the current prepared audio set with recorded
speaker results. It reuses valid completed work and schedules failed,
unattempted, missing, or obsolete results. A result is not reusable merely
because a matching filename exists. Evidence binds it to the speaker/audio
identity, audio content, Narratio-visible recognition settings, and the saved
transcript's integrity.
The relevant recognition settings remain the WhisperX service URL and language.
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
remain operational settings rather than reasons to discard completed results.
Changes to private service models or configuration remain an operator reason
to force transcription; Narratio does not inspect the service's private state.
Changes to individual audio inputs require replacement of affected results,
not automatic retranscription of unrelated unchanged speakers. New speakers
need transcription; removed speakers do not remain in the current aggregate
output set. Records must describe the inputs actually used, not grant reuse to
old bytes under a changed identity.
### Aggregate Completion, Failure, and Force
The overall stage succeeds only when every currently required speaker has a
validated, durably recorded result. Incomplete per-speaker progress does not
make the aggregate stage successful or authorize downstream consumption of an
incomplete collection. Existing dependent-stage invalidation remains in effect.
Retain bounded concurrency and the current fail-fast behavior: a speaker
failure cancels outstanding work. Results already checkpointed remain reusable
after failure, cancellation, or process interruption. A later ordinary run or
resume retries the remaining work; it does not need a new CLI command.
Checkpoint writes must be serialized so concurrently finishing speakers cannot
overwrite each other's progress. A failed persistence operation must not
advertise an unrecorded result as complete. Previously committed progress remains
the recovery authority, and checkpoint errors must be surfaced rather than
reported as successful transcription.
`--force` on transcribe regenerates all currently required speakers instead of
reusing their prior completions. Successful replacements are checkpointed too,
so an ordinary retry after a failed forced invocation can retain the newly
completed work. Old results for speakers not successfully regenerated must not
satisfy that unfinished forced replacement on the next ordinary retry.
This deliberately relaxes the current all-speakers-before-canonical-installation
implementation: individual durable results may exist while the stage remains
failed or incomplete. The all-speakers-before-stage-success boundary remains.
### Replayable WhisperX Requests
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
same request. Replay reopens the audio and reproduces the multipart boundary,
filename, fields, and framing used by the original request. Independent bodies
must not share a consumed reader or mutable read position.
Preserve bounded-memory streaming rather than buffering entire audio files in
memory. Correctly close every original and replay body and its audio resources;
preserve source-read errors and cancellation, including partial uploads and
early server responses. Successful response handling must not accidentally wait
on, or report errors solely from, an abandoned body instead of the replay that
produced the response.
Keep the existing application-level retry count, delay, classification, and
per-attempt timeout semantics. Transport replays occur within an application
attempt and its context; they are not additional Narratio-configured attempts.
Do not add another custom retry loop, silently increase retries, or disable
HTTP/2 globally as a substitute for replay support.
The adapter still validates and installs only successful JSON responses.
Request replay does not provide exactly-once server execution: a lost response
can cause repeated transcription work, as can existing application retries.
No new service-side idempotency protocol is required by this feature.
## Required Change Areas
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
evidence, with safe checkpoint updates and preservation through runner failure
handling. Review manifest validation, copying, serialization, and existing
publish/restore handling wherever the new records pass through those paths.
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
checkpoint validated completions, rebuild aggregate outputs from the current
set, and account for partial progress in ordinary and forced execution.
Review successful-stage reuse as well as failed-stage retries so stale audio
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
directory surviving cleanup. Do not introduce a second authoritative ledger
or infer completion by scanning abandoned runs.
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
an upstream service change.
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
checkpoint failure; changed/missing inputs or outputs; force followed by an
ordinary retry; concurrent successful completions; and aggregate downstream
gating. Extend existing fixtures rather than duplicating analyze's complete
test suite. Exercise actual body reconstruction after partial consumption,
multipart equivalence, cancellation/resource cleanup, and existing bounded
streaming and retry behavior—not just the presence of a callback. Default
tests remain offline and use local/fake HTTP boundaries, never paid services.
- **Documentation:** Update the canonical
[stage guide](../internal/stage-transcribe.md),
[manifest guide](../internal/manifest.md),
[WhisperX contract](../integrations/whisperx.md),
[operations guide](../operations.md), and
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
partial completion, ordinary retry versus force, and the distinction between
request replay limitations and persistent peer protocol failures.
## Compatibility and Scope Boundaries
Existing speaker identities, raw transcript formats, and the fixed pipeline
order remain unchanged. Existing manifests remain readable, but must not be
treated as containing per-speaker evidence they never recorded. Recovery relies
on verified recorded evidence, not guessed associations between old transcripts
and current audio. No general legacy-migration framework is part of the feature.
Automatic adoption of unrecorded outputs from previously failed runs is outside
this feature. Such files may be recoverable after verifying their inputs, but
their presence alone does not prove completion. No recovery/import command,
speaker-selection CLI, historical invocation ledger, new service configuration
discovery, or broader orchestration refactor is included.
The feature does not diagnose the remote service/proxy's underlying HTTP/2
problem, guarantee recovery from persistent protocol errors, or change the
policy for continuing unrelated work after failure.
## Completion Criteria
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
completed, ordinary resume retains A and B, retries C, and completes D. No
downstream stage receives an incomplete successful transcription collection.
Force replaces all four, while an ordinary retry can reuse checkpoints from
that forced replacement attempt.
The HTTP transport can reconstruct the upload after consuming the original
body, without whole-file memory buffering, lost source errors, or stranded
upload resources. A persistent peer error still fails normally after the
configured application attempts, without discarding checkpointed speakers.

View File

@@ -102,7 +102,7 @@ Safe fix:
Relevant reference: [Canonical Party Input](./integrations/party.md).
## Unexpected imported or profile value
## Unexpected imported or overlay value
Symptom:
@@ -112,21 +112,21 @@ Symptom:
Diagnostics:
```bash
narratio config sources --config /path/pipeline.yml --profile testing
narratio config sources --config /path/pipeline.yml --overlay testing
```
Add `--campaign` or `--campaign-file` when the pipeline has party-driven
artifact families. The output identifies each effective logical field's root,
import, profile, default, campaign, party, or family source without printing
import, overlay, default, campaign, party, or family source without printing
the field value or credential contents.
Safe fix:
- move a duplicated base field so it has one owner;
- correct the selected profile or its overlay; or
- correct the selected overlay; or
- correct the campaign party/family declaration that owns generated values.
To review what would actually change before switching profiles, run `config
To review what would actually change before switching overlays, run `config
diff` with the same pipeline and campaign selectors. It compares normalized
effective values rather than YAML formatting or source-file layout.
@@ -213,6 +213,170 @@ input validation.
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
## Transcription retries speakers that previously finished
Symptom:
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
finish earlier; or
- an older successful session retranscribes all speakers once after upgrade.
Likely causes:
- the earlier response or canonical file was not followed by a successful
speaker manifest checkpoint;
- the prepared audio bytes/path, WhisperX URL, or language changed;
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
no longer matches its recorded size/checksum;
- the previous invocation used `--force`, which durably cleared pre-force
speaker authority; or
- the manifest has only legacy aggregate transcribe state and cannot bind old
outputs to their source audio.
Safe fix:
- inspect the reported speaker and filesystem error, repair the prepared input
or canonical-output path when needed, then retry transcribe normally;
- allow a legacy aggregate-only record to regenerate once; and
- do not edit the manifest or copy an old JSON file into place as a substitute
for a successful checkpoint.
```bash
narratio run-stage transcribe 2026-04-04
```
Valid earlier speaker checkpoints remain reusable even while aggregate
transcribe status is failed or interrupted. Merge remains gated until
transcribe reaches aggregate success with the complete current set.
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
## WhisperX model changed but transcription was reused
Symptom:
- WhisperX model weights, prompts, or private server configuration changed at
the same configured URL, but Narratio still considers speaker transcripts
current.
Likely cause:
- Narratio can fingerprint its configured URL and language, not private state
loaded behind the service boundary.
Safe fix:
- explicitly force transcribe to establish a durable reset, then use ordinary
retry if the replacement attempt is interrupted.
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Transport replay and application retries do not provide an exactly-once remote
execution guarantee. Investigate server-side failures or duplicate work using
WhisperX diagnostics; do not infer completion from a server log without the
local speaker checkpoint.
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
[Transcribe internals](./internal/stage-transcribe.md).
## Transcription manifest checkpoint failed
Symptom:
- transcribe reports that a speaker completed but the session manifest could
not be saved.
Likely causes:
- the workspace is unwritable or out of space;
- the manifest path or parent directory is unsafe or the wrong type; or
- a transient filesystem error interrupted atomic replacement.
Safe fix:
- correct the reported filesystem or permission problem and retry transcribe
normally. Narratio reconciles from the last readable collection and reuses
only successfully checkpointed speakers.
A canonical raw transcript left by the failed checkpoint is incidental and
does not become reusable authority. Preserve the manifest and diagnostics if
the failure repeats; do not advertise the file by editing manifest evidence.
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
and [Transcribe internals](./internal/stage-transcribe.md).
## Notarius credential reference unavailable
Symptom:
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
name in `pipeline.notarius.credentials`, including during inspection or a
run that would not execute extraction.
Likely causes:
- the named environment variable is absent or empty;
- the selected secrets directory lacks a file with that exact name;
- a process environment variable exists but is empty, so bootstrap correctly
leaves it in place; or
- the Narratio reference name does not match `api_key_env` in the selected
Notarius/PromptKit profile.
Safe fix:
- set the exact nonempty environment variable or provide an equivalently named
file through the conventional or explicit secrets directory;
- remove, set to `null`, or set to `[]` when that deployment needs no credential;
and
- use `config show`, `config sources`, and `config diff` to inspect the public
reference name and ownership without exposing its value.
Narratio resolves selected credential references during ordinary pipeline
loading, before stage selection or resume. Credential changes alone do not
invalidate a completed extraction, but the current reference must still resolve
for the command to proceed.
Relevant references: [Secrets](./config.md#secrets-handling) and
[Notarius credentials](./config.md#notarius-credentials).
## Audita v2 configuration or credential failure
Symptom:
- Narratio rejects a retired `pipeline.audita` field during loading, reports
`secret.unset` or `secret.empty` for a listed credential, or Audita rejects
its external configuration or selected profile when polish runs.
Likely causes:
- the Narratio pipeline still uses v1 endpoint, model, capacity, singular key,
report, or retention settings;
- the named key is missing or empty, including a present but empty process
environment variable that takes precedence over a secrets-directory file;
- Audita's external v2 file references a backend credential name not forwarded
by `pipeline.audita.credentials`, including a backend not selected by the
current profile; or
- an Audita overlay or profile selector names a missing downstream definition.
Safe fix:
- follow the [migration table](./operations.md#migrating-audita-and-provider-credentials)
and use a supported Audita v2 executable and configuration;
- provide each needed provider-named variable through the environment or
secrets directory, and list that same name in `pipeline.audita.credentials`;
- validate the external file offline with
`audita config validate --config <file>` before switching executables; and
- inspect Audita's effective profile selection separately from Narratio's
`config show`. Narratio does not infer downstream selectors.
An exit-zero polish with a valid transcript succeeds even when Audita reports
recoverable errors or no corrections. Missing reports are allowed when Audita
reporting is disabled externally. Edits to Audita-owned files behind unchanged
selectors require `narratio run-stage polish <session-id> --force` to replace a
previously successful result. See the [Audita integration contract](./integrations/audita.md).
## Notarius executable missing
Symptom:
@@ -289,6 +453,59 @@ Do not point Notarius directly at the original source path or edit the manifest
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Notarius reference snapshot cache conflict
Symptom:
- extraction reports that a path beneath
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
non-regular or unsafe entry.
Safe fix:
- stop commands for the affected session and inspect the exact reported path;
- investigate unexpected links or special files as possible tampering; and
- remove only the reported conflicting leaf or digest directory after
confirming it belongs to the reconstructible reference cache.
A missing snapshot or corrupt regular snapshot is reconstructed automatically
from currently verified prepared input. Do not recursively delete the session
root, edit the manifest checksum, or remove cache entries while extraction is
active.
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
## Notarius resume executes work instead of reusing it
Symptom:
- Narratio executes `extract`, and Notarius makes provider calls despite
`pipeline.notarius.resume: true`.
Likely causes:
- checkpoint recording is disabled in the Notarius configuration;
- Notarius is using a nonpersistent or different checkpoint root;
- transcript/reference bytes, reference paths, pipeline/profile content,
local-backend endpoint, selected lanes, or another checkpoint identity input
changed; or
- Narratio extraction was forced, which deliberately omits `--resume`.
Safe fix:
- validate the deployed Notarius configuration and configure explicit
persistent chunk-plan and checkpoint roots;
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
and
- retry without force only when reuse is intended.
Remember that Narratio skipping a completed `extract` stage is separate from
Notarius reusing work inside an extraction that does execute. Narratio does not
read the Notarius configuration to infer checkpoint enablement.
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
and [Notarius integration](./integrations/notarius.md).
## Notarius reference selector or generated-handoff collision
Symptom:
@@ -361,10 +578,10 @@ Symptom:
Likely causes:
- the executable/config path, pipeline ID, timeout, working directory, or
configured output contracts changed;
- the pipeline ID or configured output contracts changed;
- a configured prepared reference selector, source, path, checksum, or byte
size changed;
- the direct trimmed transcript identity or bytes changed; or
- the durable bundle, index, lane set, provenance, regular-file status, or
checksum no longer validates.
@@ -374,6 +591,11 @@ Safe fix:
an unsafe path or bundle symlink error as filesystem corruption or tampering
and investigate it rather than replacing files manually.
Executable, config path, timeout, working directory, and Notarius credential
changes are operational and do not by themselves invalidate reusable extraction
evidence. A newly selected credential must nevertheless resolve during command
configuration loading.
## Notarius transitive configuration changed
Symptom:
@@ -488,18 +710,22 @@ Symptom:
or unsupported; or
- restore rejects a committed remote session manifest for the same reason.
Likely cause:
Likely causes:
- the manifest predates the supported canonical state contract. Narratio
rejects it at the schema boundary rather than interpreting aggregate output
lists or incidental files as current authority.
lists or incidental files as current authority; or
- an older strict-decoding Narratio binary is reading a current manifest that
contains additive transcribe speaker state it does not recognize.
Safe fix:
- for unsupported local state, move the old manifest aside and regenerate, or
restore a verified current committed snapshot when one is authoritative;
- for unsupported committed remote state, regenerate healthy local state as
needed and republish the session.
needed and republish the session; and
- for a downgrade rejection, return to the current binary or restore a verified
pre-upgrade backup. Do not delete only the speaker fields by hand.
```bash
narratio run 2026-04-04
@@ -511,7 +737,7 @@ old files into canonical paths merely to influence selection; availability is
manifest-owned.
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state).
and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
## Scriptorium private input changed without a rerun
@@ -717,7 +943,8 @@ Symptom:
Likely causes:
- invalid `pipeline.secrets.env_dir`;
- missing, unreadable, or invalid explicit `--secrets-dir`;
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
- missing credential environment variables;
- invalid S3 endpoint/bucket settings.
@@ -725,12 +952,12 @@ Diagnostics:
```bash
ls -la /path/to/secrets_dir
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
```
Safe fix:
- correct secret-file path and permissions;
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
- provide required env vars;
- keep secret values out of YAML.

View File

@@ -11,18 +11,26 @@ in the [configuration reference](../docs/config.md).
WhisperX URL.
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
external tools, and configured Scriptorium artifacts.
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
`conf.d` imports, a production default, and selectable production/testing
overlays. It also demonstrates canonical-party artifact families and a
testing-only disabled artifact. Validate it with:
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
one family per `artifact-families/<key>.yml`, and the fixed
`overlays/testing.yml` overlay. Production values live in the base. The
bundle demonstrates an input-only artifact dependency, two character families
sharing the campaign party, and a testing-only disabled artifact. Inspect it
offline with:
```sh
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --overlay testing
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --overlay testing
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --overlay testing
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
```
`config show` and `config sources` accept the same selectors and remain
read-only.
These inspection commands load configuration and resolve explicit credential
references, but do not load a session or create runtime state. This bundle is
intentionally local-storage and credential-reference free so the walkthrough
needs no tools or cloud services.
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
pipeline section with explanatory comments.
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
@@ -52,11 +60,24 @@ with the sample campaign and a compatible local- or S3-audio session.
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
follows the Notarius v0.6 contract. Narratio derives the players projection
follows the Notarius v0.7 contract. Narratio derives the players projection
from this party source.
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
reserves the expected filename and directory shape. Replace it with a real
FLAC file before running transcription.
The examples contain environment-variable names but no credential values. They
use fictional campaign content and reserved example domains.
The examples contain environment-variable names but no credential values. Set
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
examples that reference them. They use fictional campaign content and reserved
example domains. The Audita and Notarius names are example references, not
implicit aliases: external backend/profile configurations must use those exact
names, or the Narratio credential lists must be changed to match. See the
[Audita v2 migration guide](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/migrations/v2.md)
for maintained downstream configuration examples.
For a deployed bundle, keep credential files out of source control in a
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
environment-variable name, or select another directory with `--secrets-dir`.
Static S3 credentials are optional: omit both S3 credential references to use
the AWS SDK discovery chain.

View File

@@ -7,9 +7,8 @@ workspace:
# Optional: remove run-scoped workdir after successful publish commit.
cleanup_after_publish: false
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
# secrets:
# env_dir: ./secrets
# Credential files, when used, belong in a sibling secrets/ directory or in the
# directory selected with --secrets-dir; they are not pipeline configuration.
storage:
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
@@ -24,8 +23,8 @@ storage:
endpoint: ""
force_path_style: false
# Optional; defaults shown explicitly.
access_key_id_env: OBJECT_STORAGE_KEY_ID
secret_access_key_env: OBJECT_STORAGE_KEY
access_key_id: OBJECT_STORAGE_KEY_ID
secret_access_key: OBJECT_STORAGE_KEY
campaigns:
# Optional; defaults to /usr/local/share/narratio/campaigns.
@@ -91,22 +90,18 @@ seriatim:
filler_max_duration: 1.25
audita:
# Optional overrides; defaults shown explicitly where applicable.
# Optional overrides. Audita owns its own configuration and profiles.
binary: audita
timeout: 3h
llm_api_key_env: AUDITA_LLM_API_KEY
modules: [glossary, homophones, spoken_word, grammar]
base_url: ""
model: ""
total_llm_concurrency: 2
proposal_llm_concurrency: 1
validation_model: ""
validation_llm_concurrency: 1
# Names must match credentials on every backend in the Audita v2 config.
credentials: [AUDITA_LLM_API_KEY]
# Omit modules for Audita's default sequence; modules: [] disables corrections.
overlay: ""
proposal_profile: ""
validation_profile: ""
transcript_description: ""
config_path: /usr/local/etc/audita/config.yml
output_schema: audita-v1
work_dir_retention: auto
report: true
normalize:
# Optional; defaults shown explicitly.
@@ -129,13 +124,26 @@ trim:
report: false
notarius:
# Optional structured extraction between trim and render.
# Optional structured extraction after render and before analyze.
enabled: true
binary: notarius
# Optional environment-variable references. Match these names to the
# selected Notarius/PromptKit providers; never put credential values here.
credentials: [NOTARIUS_LLM_API_KEY]
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
# is independent of Narratio's selected overlay and may use a different name.
# See docs/config.md and docs/integrations/notarius.md for the contracts.
# config_profile: downstream-economy
config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session
timeout: 3h
working_directory: /usr/local/etc/notarius
# Reuse compatible accepted Notarius checkpoints during executed extraction.
# The Notarius config must enable recording and provide persistent cache roots.
resume: true
# auto uses or creates a source-addressed chunk plan. Forced Narratio
# extraction overrides this with refresh and disables checkpoint loading.
chunk_cache: auto
# External campaign references use prepared Narratio source IDs. Omit an
# optional binding when the selected Notarius pipeline does not need it.
references:

View File

@@ -8,8 +8,8 @@ storage:
bucket: my-dnd-archive
root_prefix: dnd
region: us-east-1
access_key_id_env: OBJECT_STORAGE_KEY_ID
secret_access_key_env: OBJECT_STORAGE_KEY
access_key_id: OBJECT_STORAGE_KEY_ID
secret_access_key: OBJECT_STORAGE_KEY
campaigns:
root: /usr/local/share/narratio/campaigns
@@ -57,14 +57,9 @@ seriatim:
audita:
binary: audita
timeout: 3h
llm_api_key_env: AUDITA_LLM_API_KEY
modules: [glossary, homophones, spoken_word, grammar]
credentials: [AUDITA_LLM_API_KEY]
# Omit modules to use Audita defaults; modules: [] disables corrections.
output_schema: audita-v1
work_dir_retention: auto
total_llm_concurrency: 2
proposal_llm_concurrency: 1
validation_llm_concurrency: 1
report: true
normalize:
output_path: transcripts/final.json

View File

@@ -1,23 +1,11 @@
scriptorium:
binary: scriptorium
config_path: ./scriptorium/config.yml
artifact_families:
character_meta:
enabled: true
for_each: party.characters
prompt_id: dnd.character_meta
output_path_pattern: artifacts/characters/{character_id}/meta.md
member_vars:
character_id: character_id
character_name: character.name
player_name: player.name
class_summary: character.class_summary
character_items:
enabled: true
for_each: party.characters
prompt_id: dnd.character_items
profile_id: production-placeholder
output_path_pattern: artifacts/characters/{character_id}/items.md
member_dependencies: [character_meta]
inputs:
character_meta:
source: narratio.member_artifact.character_meta

View File

@@ -0,0 +1,13 @@
scriptorium:
artifact_families:
character_meta:
enabled: true
for_each: party.characters
prompt_id: dnd.character_meta
profile_id: production-placeholder
output_path_pattern: artifacts/characters/{character_id}/meta.md
member_vars:
character_id: character_id
character_name: character.name
player_name: player.name
class_summary: character.class_summary

View File

@@ -0,0 +1,11 @@
scriptorium:
artifacts:
player_handout:
enabled: true
prompt_id: dnd.player_handout
profile_id: production-placeholder
output_path: artifacts/player-handout.md
inputs:
recap:
source: narratio.artifact.session_recap
required: true

View File

@@ -0,0 +1,11 @@
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
profile_id: production-placeholder
output_path: artifacts/session-recap.md
inputs:
transcript:
source: narratio.transcript.final_trimmed
required: true

View File

@@ -10,6 +10,8 @@ audita:
binary: audita
modules: [glossary, grammar]
output_schema: audita-v1
proposal_profile: narratio-production-model-placeholder
validation_profile: narratio-production-validator-placeholder
normalize:
output_path: transcripts/final.json

View File

@@ -1,9 +1,13 @@
audita:
model: narratio-testing-model-placeholder
validation_model: narratio-testing-validator-placeholder
proposal_profile: narratio-testing-model-placeholder
validation_profile: narratio-testing-validator-placeholder
scriptorium:
artifacts:
session_recap:
profile_id: testing-placeholder
player_handout:
profile_id: testing-placeholder
testing_notes:
enabled: false
output_path: artifacts/testing-notes.md

View File

@@ -1,15 +1,14 @@
# Copyable production/testing pipeline entry point. Every fragment is named
# explicitly; Narratio never scans conf.d automatically.
composition:
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
# these explicit path and glob imports; Narratio never scans directories itself.
configkit:
imports:
- conf.d/platform.yml
- conf.d/storage.yml
- conf.d/transcript.yml
- conf.d/artifacts.yml
- conf.d/publish.yml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
- path: conf.d/platform.yml
- path: conf.d/storage.yml
- path: conf.d/transcript.yml
- path: conf.d/publish.yml
- glob: artifacts/*.yml
- glob: artifact-families/*.yml
scriptorium:
binary: scriptorium
config_path: ./scriptorium/config.yml

View File

@@ -1,10 +0,0 @@
audita:
model: narratio-production-model-placeholder
validation_model: narratio-production-validator-placeholder
scriptorium:
artifact_families:
character_meta:
profile_id: production-placeholder
character_items:
profile_id: production-placeholder

2
go.mod
View File

@@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
go 1.26.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.4.0
github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
@@ -28,4 +29,5 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
)

4
go.sum
View File

@@ -1,3 +1,5 @@
gitea.maximumdirect.net/eric/ConfigKit v0.4.0 h1:fo+vKff3zTGeWqJIA38NtT84b6OUP//9ZmBH3lFuKgk=
gitea.maximumdirect.net/eric/ConfigKit v0.4.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
@@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=

View File

@@ -17,13 +17,11 @@ func (n *NoopRunner) Run(ctx context.Context, req PolishRequest) (PolishResult,
if err := ctx.Err(); err != nil {
return PolishResult{}, err
}
if err := materializePlaceholders(req); err != nil {
if err := materializePlaceholders(req, false); err != nil {
return PolishResult{}, err
}
return PolishResult{
ProcessedTranscriptPath: req.OutputProcessedPath,
ReportPath: req.ReportPath,
WorkDir: req.WorkDir,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
@@ -47,18 +45,20 @@ func (f *FakeRunner) Run(ctx context.Context, req PolishRequest) (PolishResult,
if f.Err != nil {
return PolishResult{}, f.Err
}
if err := materializePlaceholders(req); err != nil {
if err := materializePlaceholders(req, true); err != nil {
return PolishResult{}, err
}
res := f.Result
if res.ProcessedTranscriptPath == "" {
res.ProcessedTranscriptPath = req.OutputProcessedPath
}
if res.ReportPath == "" {
res.ReportPath = req.ReportPath
}
if res.WorkDir == "" {
res.WorkDir = req.WorkDir
if res.Reports == (ReportBundle{}) {
dir := filepath.Dir(req.OutputProcessedPath)
res.Reports = ReportBundle{
ErrorsPath: filepath.Join(dir, "errors.yml"),
RecoveriesPath: filepath.Join(dir, "recoveries.yml"),
OverviewPath: filepath.Join(dir, "report.md"),
}
}
if res.StdoutLogPath == "" {
res.StdoutLogPath = req.StdoutLogPath
@@ -75,10 +75,10 @@ func (f *FakeRunner) Run(ctx context.Context, req PolishRequest) (PolishResult,
return res, nil
}
func materializePlaceholders(req PolishRequest) error {
func materializePlaceholders(req PolishRequest, reports bool) error {
if req.GeneratedConfigPath != "" {
payload := map[string]any{
"schema": "audita.generated.v1",
"schema": "audita.generated.v2",
"placeholder": true,
"merged_transcript_path": req.MergedTranscriptPath,
"output_path": req.OutputProcessedPath,
@@ -103,11 +103,18 @@ func materializePlaceholders(req PolishRequest) error {
}); err != nil {
return err
}
if err := writeJSONIfRequested(req.ReportPath, map[string]any{
"schema": "audita.report.v1",
"steps": []any{},
}); err != nil {
return err
if !reports {
return nil
}
for _, report := range []struct{ name, contents string }{
{"errors.yml", "schema: audita-errors\nversion: 1\nstatus: completed\nerrors: []\n"},
{"recoveries.yml", "schema: audita-recoveries\nversion: 1\nstatus: completed\nrecoveries: []\n"},
{"report.md", "# Audita report\n"},
} {
path := filepath.Join(filepath.Dir(req.OutputProcessedPath), report.name)
if err := fileops.WriteFileAtomic(path, []byte(report.contents), fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("write report %q: %w", path, err)
}
}
return nil
}

View File

@@ -11,24 +11,29 @@ type Runner interface {
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
}
// PolishRequest describes an audita invocation.
// PolishRequest describes an Audita invocation. Nil Modules uses the runner's
// configured selection; a non-nil empty slice selects no modules.
type PolishRequest struct {
GeneratedConfigPath string
MergedTranscriptPath string
OutputProcessedPath string
GlossaryPath string
ReportPath string
WorkDir string
Modules []string
StdoutLogPath string
StderrLogPath string
}
// PolishResult describes a polish output.
// ReportBundle contains optional reports adjacent to the requested output.
type ReportBundle struct {
ErrorsPath string
RecoveriesPath string
OverviewPath string
}
// PolishResult describes a polish output and available diagnostics.
type PolishResult struct {
ProcessedTranscriptPath string
ReportPath string
WorkDir string
Reports ReportBundle
StdoutLogPath string
StderrLogPath string
GeneratedConfigPath string

View File

@@ -3,13 +3,14 @@ package audita
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"strconv"
"path/filepath"
"strings"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
)
@@ -17,89 +18,27 @@ import (
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
// MaxReportOutputBytes bounds Audita's optional report JSON result.
// MaxReportOutputBytes bounds each optional Audita report.
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
type SubprocessRunnerConfig struct {
Binary string
Timeout time.Duration
LLMAPIKeyEnv string
Modules []string
BaseURL string
Model string
TranscriptDescription string
ConfigPath string
OutputSchema string
WorkDirRetention string
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
ValidationModel string
ValidationLLMConcurrency *int
Report bool
Binary string
Timeout time.Duration
Credentials []configkit.Secret
Modules []string
Overlay string
ProposalProfile string
ValidationProfile string
TranscriptDescription string
ConfigPath string
OutputSchema string
}
// SubprocessRunner invokes Audita via subprocess.
type SubprocessRunner struct {
binary string
timeout time.Duration
llmAPIKeyEnv string
modules []string
baseURL string
model string
transcriptDescription string
configPath string
outputSchema string
workDirRetention string
totalLLMConcurrency *int
proposalLLMConcurrency *int
validationModel string
validationLLMConcurrency *int
report bool
}
// NewSubprocessRunnerFromConfigValues parses config-derived values once.
func NewSubprocessRunnerFromConfigValues(
binary string,
timeout string,
llmAPIKeyEnv string,
modules []string,
baseURL string,
model string,
transcriptDescription string,
configPath string,
outputSchema string,
workDirRetention string,
totalLLMConcurrency *int,
proposalLLMConcurrency *int,
validationModel string,
validationLLMConcurrency *int,
report bool,
) (*SubprocessRunner, error) {
if strings.TrimSpace(timeout) == "" {
return nil, fmt.Errorf("audita timeout is required")
}
parsedTimeout, err := time.ParseDuration(timeout)
if err != nil {
return nil, fmt.Errorf("parse audita timeout %q: %w", timeout, err)
}
return NewSubprocessRunner(SubprocessRunnerConfig{
Binary: binary,
Timeout: parsedTimeout,
LLMAPIKeyEnv: llmAPIKeyEnv,
Modules: modules,
BaseURL: baseURL,
Model: model,
TranscriptDescription: transcriptDescription,
ConfigPath: configPath,
OutputSchema: outputSchema,
WorkDirRetention: workDirRetention,
TotalLLMConcurrency: totalLLMConcurrency,
ProposalLLMConcurrency: proposalLLMConcurrency,
ValidationModel: validationModel,
ValidationLLMConcurrency: validationLLMConcurrency,
Report: report,
})
cfg SubprocessRunnerConfig
run func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
}
// NewSubprocessRunner constructs a validated Audita subprocess runner.
@@ -115,278 +54,172 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
return nil, fmt.Errorf("audita module at index %d is empty", i)
}
}
if strings.TrimSpace(cfg.BaseURL) != "" {
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil {
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
}
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
}
}
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
}
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
}
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
}
switch strings.TrimSpace(cfg.OutputSchema) {
case "", "bare-segments", "audita-v1":
default:
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
}
switch strings.TrimSpace(cfg.WorkDirRetention) {
case "", "always", "auto", "never":
default:
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
cfg.Binary = strings.TrimSpace(cfg.Binary)
cfg.Credentials = append([]configkit.Secret(nil), cfg.Credentials...)
cfg.Modules = cloneModules(cfg.Modules)
for i := range cfg.Modules {
cfg.Modules[i] = strings.TrimSpace(cfg.Modules[i])
}
modules := make([]string, len(cfg.Modules))
for i, m := range cfg.Modules {
modules[i] = strings.TrimSpace(m)
}
return &SubprocessRunner{
binary: strings.TrimSpace(cfg.Binary),
timeout: cfg.Timeout,
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
modules: modules,
baseURL: strings.TrimSpace(cfg.BaseURL),
model: strings.TrimSpace(cfg.Model),
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
configPath: strings.TrimSpace(cfg.ConfigPath),
outputSchema: strings.TrimSpace(cfg.OutputSchema),
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
totalLLMConcurrency: cfg.TotalLLMConcurrency,
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
validationModel: strings.TrimSpace(cfg.ValidationModel),
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
report: cfg.Report,
}, nil
return &SubprocessRunner{cfg: cfg, run: subprocess.Run}, nil
}
// Run executes Audita process with deterministic flags and validates output artifacts.
func cloneModules(modules []string) []string {
if modules == nil {
return nil
}
return append([]string{}, modules...)
}
// Run executes Audita process and validates its corrected transcript.
func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishResult, error) {
if r == nil {
return PolishResult{}, fmt.Errorf("audita subprocess runner is nil")
}
if strings.TrimSpace(req.MergedTranscriptPath) == "" {
return PolishResult{}, fmt.Errorf("audita merged transcript path is required")
}
if strings.TrimSpace(req.GlossaryPath) == "" {
return PolishResult{}, fmt.Errorf("audita glossary path is required")
}
if strings.TrimSpace(req.OutputProcessedPath) == "" {
return PolishResult{}, fmt.Errorf("audita output processed path is required")
}
if strings.TrimSpace(req.WorkDir) == "" {
return PolishResult{}, fmt.Errorf("audita work dir is required")
}
if r.report && strings.TrimSpace(req.ReportPath) == "" {
return PolishResult{}, fmt.Errorf("audita report is enabled but report path is missing")
}
reqModules := req.Modules
if reqModules == nil {
reqModules = append([]string(nil), r.modules...)
}
args := r.buildArgs(req, reqModules)
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
credentialPresent := false
env := map[string]string{}
if credentialEnvVar != "" {
credential, ok := os.LookupEnv(credentialEnvVar)
if !ok || strings.TrimSpace(credential) == "" {
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
for _, field := range []struct{ name, value string }{
{"merged transcript path", req.MergedTranscriptPath},
{"glossary path", req.GlossaryPath},
{"output processed path", req.OutputProcessedPath},
} {
if strings.TrimSpace(field.value) == "" {
return PolishResult{}, fmt.Errorf("audita %s is required", field.name)
}
env["AUDITA_LLM_API_KEY"] = credential
credentialPresent = true
}
modules := req.Modules
if modules == nil {
modules = r.cfg.Modules
}
for i, module := range modules {
if strings.TrimSpace(module) == "" {
return PolishResult{}, fmt.Errorf("audita module at index %d is empty", i)
}
}
modules = cloneModules(modules)
for i := range modules {
modules[i] = strings.TrimSpace(modules[i])
}
args := r.buildArgs(req, modules)
env, names, err := subprocess.CredentialEnvironment(r.cfg.Credentials)
if err != nil {
return PolishResult{}, fmt.Errorf("audita credentials: %w", err)
}
if req.GeneratedConfigPath != "" {
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
if err := r.writeInvocationConfig(req, args, modules, names); err != nil {
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
}
}
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
Executable: r.binary,
Args: args,
Timeout: r.timeout,
EnvOverrides: env,
DiagnosticOwner: "audita",
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
run := r.run
if run == nil {
run = subprocess.Run
}
runRes, runErr := run(ctx, subprocess.RunRequest{
Executable: r.cfg.Binary, Args: args, Timeout: r.cfg.Timeout,
EnvOverrides: env, SensitiveEnvNames: names, DiagnosticOwner: "audita",
StdoutLogPath: req.StdoutLogPath, StderrLogPath: req.StderrLogPath,
})
if err != nil {
wrappedMessage := fmt.Sprintf(
"run audita process (binary=%q, stdout_log=%q, stderr_log=%q)",
r.binary,
req.StdoutLogPath,
req.StderrLogPath,
)
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
"%s: %w",
wrappedMessage,
err,
)
result := r.result(req, modules, names, runRes)
if runErr != nil {
result.Reports, _ = discoverReports(req.OutputProcessedPath)
return result, fmt.Errorf("run audita process (binary=%q, output_dir=%q, stdout_log=%q, stderr_log=%q): %w",
r.cfg.Binary, filepath.Dir(req.OutputProcessedPath), req.StdoutLogPath, req.StderrLogPath, runErr)
}
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
result.Reports, _ = discoverReports(req.OutputProcessedPath)
return result, fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
}
if r.report {
if err := validateJSONFile(req.ReportPath); err != nil {
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
}
reports, err := discoverReports(req.OutputProcessedPath)
if err != nil {
return result, err
}
return PolishResult{
ProcessedTranscriptPath: req.OutputProcessedPath,
ReportPath: req.ReportPath,
WorkDir: req.WorkDir,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
ExitCode: runRes.ExitCode,
Duration: runRes.Duration,
InvokedBinary: r.binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": reqModules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
},
}, nil
result.Reports = reports
return result, nil
}
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
func (r *SubprocessRunner) result(req PolishRequest, modules, names []string, runRes subprocess.RunResult) PolishResult {
return PolishResult{
ProcessedTranscriptPath: req.OutputProcessedPath,
ReportPath: req.ReportPath,
WorkDir: req.WorkDir,
StdoutLogPath: req.StdoutLogPath,
StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath,
ExitCode: runRes.ExitCode,
Duration: runRes.Duration,
InvokedBinary: r.binary,
StdoutLogPath: req.StdoutLogPath, StderrLogPath: req.StderrLogPath,
GeneratedConfigPath: req.GeneratedConfigPath, ExitCode: runRes.ExitCode,
Duration: runRes.Duration, InvokedBinary: r.cfg.Binary,
Metadata: map[string]any{
"adapter": "audita_subprocess",
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
"adapter": "audita_subprocess", "modules": modules,
"overlay": r.cfg.Overlay, "proposal_profile": r.cfg.ProposalProfile,
"validation_profile": r.cfg.ValidationProfile,
"transcript_description": r.cfg.TranscriptDescription,
"config_path": r.cfg.ConfigPath, "output_schema": r.cfg.OutputSchema,
"credential_names": names,
},
}
}
func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []string {
args := []string{
"process",
req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--work-dir", req.WorkDir,
}
if r.baseURL != "" {
args = append(args, "--base-url", r.baseURL)
}
if r.model != "" {
args = append(args, "--model", r.model)
}
if len(modules) > 0 {
args := []string{"process", req.MergedTranscriptPath, "--glossary", req.GlossaryPath, "--output", req.OutputProcessedPath}
if modules != nil {
args = append(args, "--modules", strings.Join(modules, ","))
}
if r.report {
args = append(args, "--report-json", req.ReportPath)
}
if r.transcriptDescription != "" {
args = append(args, "--transcript-description", r.transcriptDescription)
}
if r.configPath != "" {
args = append(args, "--config", r.configPath)
}
if r.outputSchema != "" {
args = append(args, "--output-schema", r.outputSchema)
}
if r.workDirRetention != "" {
args = append(args, "--work-dir-retention", r.workDirRetention)
}
if r.totalLLMConcurrency != nil {
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
}
if r.proposalLLMConcurrency != nil {
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
}
if r.validationModel != "" {
args = append(args, "--validation-model", r.validationModel)
}
if r.validationLLMConcurrency != nil {
args = append(args, "--validation-llm-concurrency", strconv.Itoa(*r.validationLLMConcurrency))
for _, setting := range []struct{ flag, value string }{
{"--config", r.cfg.ConfigPath}, {"--overlay", r.cfg.Overlay},
{"--proposal-profile", r.cfg.ProposalProfile}, {"--validation-profile", r.cfg.ValidationProfile},
{"--transcript-description", r.cfg.TranscriptDescription}, {"--output-schema", r.cfg.OutputSchema},
} {
if setting.value != "" {
args = append(args, setting.flag, setting.value)
}
}
return args
}
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
payload := map[string]any{
"schema": "audita.generated.v1",
"binary": r.binary,
"args": args,
"timeout": r.timeout.String(),
"modules": modules,
"base_url": r.baseURL,
"model": r.model,
"transcript_description": r.transcriptDescription,
"config_path": r.configPath,
"output_schema": r.outputSchema,
"work_dir_retention": r.workDirRetention,
"validation_model": r.validationModel,
"total_llm_concurrency": r.totalLLMConcurrency,
"proposal_llm_concurrency": r.proposalLLMConcurrency,
"validation_llm_concurrency": r.validationLLMConcurrency,
"report_enabled": r.report,
"merged_transcript_path": req.MergedTranscriptPath,
"glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath,
"report_path": req.ReportPath,
"work_dir": req.WorkDir,
"credential_env_var": r.llmAPIKeyEnv,
"credential_present": credentialPresent,
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args, modules, names []string) error {
return writeGeneratedYAML(req.GeneratedConfigPath, map[string]any{
"schema": "audita.generated.v2", "binary": r.cfg.Binary, "args": args,
"timeout": r.cfg.Timeout.String(), "modules": modules,
"overlay": r.cfg.Overlay, "proposal_profile": r.cfg.ProposalProfile,
"validation_profile": r.cfg.ValidationProfile,
"transcript_description": r.cfg.TranscriptDescription,
"config_path": r.cfg.ConfigPath, "output_schema": r.cfg.OutputSchema,
"merged_transcript_path": req.MergedTranscriptPath, "glossary_path": req.GlossaryPath,
"output_path": req.OutputProcessedPath, "credential_names": names,
})
}
func discoverReports(outputPath string) (ReportBundle, error) {
var reports ReportBundle
for _, report := range []struct {
name string
dest *string
}{
{"errors.yml", &reports.ErrorsPath},
{"recoveries.yml", &reports.RecoveriesPath},
{"report.md", &reports.OverviewPath},
} {
path := filepath.Join(filepath.Dir(outputPath), report.name)
info, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
return reports, fmt.Errorf("inspect audita report %q: %w", path, err)
}
if !info.Mode().IsRegular() {
return reports, fmt.Errorf("audita report %q is not a regular file", path)
}
if _, err := fileops.ReadFileBounded(path, MaxReportOutputBytes); err != nil {
return reports, fmt.Errorf("audita report %q exceeds or cannot be read within %d-byte limit: %w", path, MaxReportOutputBytes, err)
}
*report.dest = path
}
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
return reports, nil
}
func validateProcessedOutput(path string) error {
data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
data, err := fileops.ReadFileBounded(path, MaxProcessedOutputBytes)
if err != nil {
return err
return fmt.Errorf("audita processed transcript result exceeds or cannot be read within %d-byte limit: %w", MaxProcessedOutputBytes, err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
return fmt.Errorf("parse json: %w", err)
@@ -400,34 +233,3 @@ func validateProcessedOutput(path string) error {
}
return nil
}
func addSubprocessStreamHint(message string, runErr error) string {
if runErr == nil {
return message
}
lower := strings.ToLower(runErr.Error())
if strings.Contains(lower, "bad file descriptor") || strings.Contains(lower, "exit code 120") {
return message + "; hint=audita child process may have started with invalid stderr/stdout descriptors"
}
return message
}
func validateJSONFile(path string) error {
data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
if err != nil {
return err
}
var v any
if err := json.Unmarshal(data, &v); err != nil {
return fmt.Errorf("parse json: %w", err)
}
return nil
}
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
data, err := fileops.ReadFileBounded(path, limit)
if err != nil {
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
}
return data, nil
}

View File

@@ -2,665 +2,262 @@ package audita
import (
"context"
"encoding/json"
"fmt"
"errors"
"os"
"path/filepath"
"runtime"
"reflect"
"strings"
"testing"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
)
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
wrapper := writeAuditaHelperWrapper(t)
totalLLMConcurrency := 3
proposalLLMConcurrency := 2
validationLLMConcurrency := 2
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
Binary: wrapper,
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary", "homophones", "glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
TranscriptDescription: "Campaign Session 42",
ConfigPath: "/etc/audita/config.yml",
OutputSchema: "audita-v1",
WorkDirRetention: "auto",
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
ValidationModel: "openrouter/google/gemma-4-31b-it",
ValidationLLMConcurrency: &validationLLMConcurrency,
Report: true,
})
if err != nil {
t.Fatalf("NewSubprocessRunner() error = %v", err)
}
func auditaTestRequest(t *testing.T) PolishRequest {
t.Helper()
dir := t.TempDir()
req := PolishRequest{
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
MergedTranscriptPath: filepath.Join(dir, "base.json"),
return PolishRequest{
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
GlossaryPath: filepath.Join(dir, "glossary.yml"),
OutputProcessedPath: filepath.Join(dir, "polished.json"),
ReportPath: filepath.Join(dir, "audita.report.json"),
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
}
writeAuditaTestFile(t, req.MergedTranscriptPath, `{"segments":[]}`)
writeAuditaTestFile(t, req.GlossaryPath, "terms: []\n")
res, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if res.ProcessedTranscriptPath != req.OutputProcessedPath {
t.Fatalf("ProcessedTranscriptPath = %q, want %q", res.ProcessedTranscriptPath, req.OutputProcessedPath)
}
if res.ReportPath != req.ReportPath {
t.Fatalf("ReportPath = %q, want %q", res.ReportPath, req.ReportPath)
}
if res.WorkDir != req.WorkDir {
t.Fatalf("WorkDir = %q, want %q", res.WorkDir, req.WorkDir)
}
if res.InvokedBinary != wrapper {
t.Fatalf("InvokedBinary = %q, want %q", res.InvokedBinary, wrapper)
}
if res.ExitCode != 0 {
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
}
if res.Duration <= 0 {
t.Fatalf("Duration = %s, want > 0", res.Duration)
}
assertJSONFileAudita(t, req.OutputProcessedPath)
assertJSONFileAudita(t, req.ReportPath)
if _, err := os.Stat(req.StdoutLogPath); err != nil {
t.Fatalf("stdout log missing: %v", err)
}
if _, err := os.Stat(req.StderrLogPath); err != nil {
t.Fatalf("stderr log missing: %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
wantArgs := []string{
"process", req.MergedTranscriptPath,
"--glossary", req.GlossaryPath,
"--output", req.OutputProcessedPath,
"--work-dir", req.WorkDir,
"--base-url", "https://openrouter.ai/api/v1",
"--model", "openrouter/google/gemma-4-31b-it",
"--modules", "glossary,homophones,glossary",
"--report-json", req.ReportPath,
"--transcript-description", "Campaign Session 42",
"--config", "/etc/audita/config.yml",
"--output-schema", "audita-v1",
"--work-dir-retention", "auto",
"--total-llm-concurrency", "3",
"--proposal-llm-concurrency", "2",
"--validation-model", "openrouter/google/gemma-4-31b-it",
"--validation-llm-concurrency", "2",
}
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
}
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
}
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
}
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
if err != nil {
t.Fatalf("read generated config: %v", err)
}
if strings.Contains(string(cfgData), "super-secret") {
t.Fatalf("generated config must not contain credential value")
}
}
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
t.Fatalf("error = %q, want missing credential guidance", err.Error())
}
}
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
res, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if res.Metadata["credential_present"] != false {
t.Fatalf("credential_present = %#v, want false", res.Metadata["credential_present"])
}
rec := readAuditaHelperRecord(t, recordPath)
if rec.Env["AUDITA_LLM_API_KEY"] != "" {
t.Fatalf("AUDITA_LLM_API_KEY = %q, want omitted/empty", rec.Env["AUDITA_LLM_API_KEY"])
}
}
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
LLMAPIKeyEnv: "",
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
}
}
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--modules" {
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "success")
recordPath := filepath.Join(t.TempDir(), "record.json")
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "",
Report: false,
})
req := auditaReqForTest(t, false)
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
rec := readAuditaHelperRecord(t, recordPath)
for i := 0; i < len(rec.Args); i++ {
if rec.Args[i] == "--base-url" {
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
}
if rec.Args[i] == "--model" {
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
}
}
}
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "fail")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "run audita process") {
t.Fatalf("error = %q, want subprocess context", err.Error())
}
if !strings.Contains(err.Error(), "exit code") {
t.Fatalf("error = %q, want exit code context", err.Error())
}
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
}
}
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "hint=audita child process may have started with invalid stderr/stdout descriptors") {
t.Fatalf("error = %q, want stderr descriptor hint", err.Error())
}
}
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "validate audita processed output") {
t.Fatalf("error = %q, want output validation context", err.Error())
}
}
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "parse json") {
t.Fatalf("error = %q, want parse json context", err.Error())
}
}
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: false,
})
req := auditaReqForTest(t, false)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "segments") {
t.Fatalf("error = %q, want segments validation context", err.Error())
}
}
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
Binary: writeAuditaHelperWrapper(t),
Timeout: mustParseAuditaDuration(t, "2s"),
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
Modules: []string{"glossary"},
BaseURL: "https://openrouter.ai/api/v1",
Model: "openrouter/google/gemma-4-31b-it",
Report: true,
})
req := auditaReqForTest(t, true)
_, err := runner.Run(context.Background(), req)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "validate audita report output") {
t.Fatalf("error = %q, want report validation context", err.Error())
}
}
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected binary validation error")
}
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
if err == nil {
t.Fatal("expected timeout parse error")
}
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
Binary: "audita",
Timeout: mustParseAuditaDuration(t, "1s"),
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
Modules: []string{"glossary"},
BaseURL: "://",
Model: "openrouter/google/gemma-4-31b-it",
})
if err == nil {
t.Fatal("expected base url validation error")
}
}
type auditaHelperRecord struct {
Args []string `json:"args"`
Env map[string]string `json:"env"`
}
func TestAuditaSubprocessHelper(t *testing.T) {
if os.Getenv("GO_WANT_AUDITA_HELPER") != "1" {
return
}
args := os.Args
start := -1
for i := range args {
if args[i] == "--" {
start = i + 1
break
}
}
if start < 0 || start >= len(args) {
_, _ = os.Stderr.WriteString("missing -- args separator\n")
os.Exit(2)
}
procArgs := args[start:]
outPath := auditaFlagValue(procArgs, "--output")
reportPath := auditaFlagValue(procArgs, "--report-json")
recordPath := os.Getenv("AUDITA_HELPER_RECORD_PATH")
if strings.TrimSpace(recordPath) != "" {
rec := auditaHelperRecord{
Args: procArgs,
Env: map[string]string{
"AUDITA_LLM_API_KEY": os.Getenv("AUDITA_LLM_API_KEY"),
"AUDITA_LLM_CONCURRENCY": os.Getenv("AUDITA_LLM_CONCURRENCY"),
"AUDITA_INHERITED_MARKER": os.Getenv("AUDITA_INHERITED_MARKER"),
},
}
data, _ := json.Marshal(rec)
_ = os.MkdirAll(filepath.Dir(recordPath), 0o755)
_ = os.WriteFile(recordPath, data, 0o644)
}
mode := os.Getenv("AUDITA_HELPER_MODE")
switch mode {
case "success":
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1","segments":[]}`)
if reportPath != "" {
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
}
_, _ = os.Stdout.WriteString("audita helper success stdout\n")
_, _ = os.Stderr.WriteString("audita helper success stderr\n")
os.Exit(0)
case "fail":
_, _ = os.Stderr.WriteString("audita helper failure\n")
os.Exit(8)
case "fail_bad_fd":
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
os.Exit(120)
case "missing_output":
if reportPath != "" {
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
}
os.Exit(0)
case "invalid_output":
writeAuditaHelperFile(outPath, `not-json`)
if reportPath != "" {
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
}
os.Exit(0)
case "segments_missing":
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1"}`)
if reportPath != "" {
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
}
os.Exit(0)
case "invalid_report":
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1","segments":[]}`)
if reportPath != "" {
writeAuditaHelperFile(reportPath, `not-json`)
}
os.Exit(0)
default:
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
os.Exit(2)
}
}
func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunner {
t.Helper()
r, err := NewSubprocessRunner(cfg)
if err != nil {
t.Fatalf("NewSubprocessRunner() error = %v", err)
}
return r
}
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
t.Helper()
dir := t.TempDir()
merged := filepath.Join(dir, "base.json")
glossary := filepath.Join(dir, "glossary.yml")
writeAuditaTestFile(t, merged, `{"segments":[]}`)
writeAuditaTestFile(t, glossary, "terms: []\n")
req := PolishRequest{
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
MergedTranscriptPath: merged,
GlossaryPath: glossary,
OutputProcessedPath: filepath.Join(dir, "polished.json"),
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
StdoutLogPath: filepath.Join(dir, "stdout.log"),
StderrLogPath: filepath.Join(dir, "stderr.log"),
}
if withReport {
req.ReportPath = filepath.Join(dir, "audita.report.json")
}
return req
}
func writeAuditaHelperWrapper(t *testing.T) string {
func auditaTestRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunner {
t.Helper()
exe, err := os.Executable()
if cfg.Binary == "" {
cfg.Binary = "audita"
}
if cfg.Timeout == 0 {
cfg.Timeout = time.Second
}
runner, err := NewSubprocessRunner(cfg)
if err != nil {
t.Fatalf("os.Executable() error = %v", err)
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "audita-helper-wrapper.sh")
content := "#!/bin/sh\nexec \"" + exe + "\" -test.run=TestAuditaSubprocessHelper -- \"$@\"\n"
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
return path
return runner
}
func writeAuditaTestFile(t *testing.T, path, contents string) {
func auditaTestSecret(t *testing.T, name, value string) configkit.Secret {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", path, err)
t.Setenv(name, value)
type loaded struct {
Credential configkit.Secret `yaml:"credential"`
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatalf("WriteFile(%q): %v", path, err)
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+name+"\n")), configkit.Options[loaded]{})
if err != nil {
t.Fatal(err)
}
return result.Config.Credential
}
func writeAuditaTestFile(t *testing.T, path, value string) {
t.Helper()
if err := os.WriteFile(path, []byte(value), 0o600); err != nil {
t.Fatal(err)
}
}
func writeAuditaHelperFile(path, contents string) {
if strings.TrimSpace(path) == "" {
return
}
_ = os.MkdirAll(filepath.Dir(path), 0o755)
_ = os.WriteFile(path, []byte(contents), 0o644)
}
func auditaFlagValue(args []string, name string) string {
for i := 0; i < len(args)-1; i++ {
if args[i] == name {
return args[i+1]
func TestSubprocessRunnerUsesV2InvocationAndCapturedCredentials(t *testing.T) {
const first = "NARRATIO_AUDITA_CUSTOM_ONE"
const second = "NARRATIO_AUDITA_CUSTOM_TWO"
firstSecret := auditaTestSecret(t, first, "loaded-one")
secondSecret := auditaTestSecret(t, second, "loaded-two")
t.Setenv(first, "parent-changed")
t.Setenv(second, "parent-changed-too")
t.Setenv("NARRATIO_AUDITA_UNSELECTED", "not-forwarded")
runner := auditaTestRunner(t, SubprocessRunnerConfig{
Credentials: []configkit.Secret{firstSecret, secondSecret},
Modules: []string{"grammar", "glossary", "grammar"},
Overlay: "testing", ProposalProfile: "proposal", ValidationProfile: "validation",
TranscriptDescription: "campaign", ConfigPath: "/etc/audita/config.yml", OutputSchema: "audita-v1",
})
req := auditaTestRequest(t)
var captured subprocess.RunRequest
runner.run = func(_ context.Context, processReq subprocess.RunRequest) (subprocess.RunResult, error) {
captured = processReq
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
writeAuditaTestFile(t, filepath.Join(filepath.Dir(req.OutputProcessedPath), name), "recoverable diagnostic\n")
}
return subprocess.RunResult{}, nil
}
return ""
}
func readAuditaHelperRecord(t *testing.T, path string) auditaHelperRecord {
t.Helper()
data, err := os.ReadFile(path)
result, err := runner.Run(context.Background(), req)
if err != nil {
t.Fatalf("ReadFile(%q): %v", path, err)
t.Fatal(err)
}
var rec auditaHelperRecord
if err := json.Unmarshal(data, &rec); err != nil {
t.Fatalf("json unmarshal helper record: %v", err)
want := []string{"process", req.MergedTranscriptPath, "--glossary", req.GlossaryPath, "--output", req.OutputProcessedPath,
"--modules", "grammar,glossary,grammar", "--config", "/etc/audita/config.yml", "--overlay", "testing",
"--proposal-profile", "proposal", "--validation-profile", "validation", "--transcript-description", "campaign", "--output-schema", "audita-v1"}
if !reflect.DeepEqual(captured.Args, want) {
t.Fatalf("args = %#v, want %#v", captured.Args, want)
}
return rec
}
func assertJSONFileAudita(t *testing.T, path string) {
t.Helper()
data, err := os.ReadFile(path)
if len(captured.EnvOverrides) != 2 || captured.EnvOverrides[first] != "loaded-one" || captured.EnvOverrides[second] != "loaded-two" || captured.EnvOverrides["NARRATIO_AUDITA_UNSELECTED"] != "" {
t.Fatalf("credential forwarding = %#v", captured.EnvOverrides)
}
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{first, second}) {
t.Fatalf("sensitive names = %#v", captured.SensitiveEnvNames)
}
if result.Reports.ErrorsPath == "" || result.Reports.RecoveriesPath == "" || result.Reports.OverviewPath == "" {
t.Fatalf("reports = %#v", result.Reports)
}
generated, err := os.ReadFile(req.GeneratedConfigPath)
if err != nil {
t.Fatalf("ReadFile(%q): %v", path, err)
t.Fatal(err)
}
var v any
if err := json.Unmarshal(data, &v); err != nil {
t.Fatalf("json unmarshal %q: %v", path, err)
if !strings.Contains(string(generated), "audita.generated.v2") || !strings.Contains(string(generated), first) || strings.Contains(string(generated), "loaded-one") {
t.Fatalf("unsafe or incomplete generated invocation: %s", generated)
}
}
func mustParseAuditaDuration(t *testing.T, value string) time.Duration {
t.Helper()
d, err := time.ParseDuration(value)
if err != nil {
t.Fatalf("time.ParseDuration(%q) error = %v", value, err)
func TestSubprocessRunnerModulePresenceAndReportAbsence(t *testing.T) {
for _, test := range []struct {
name string
configured []string
requested []string
wantFlag bool
wantValue string
}{
{"omitted", nil, nil, false, ""},
{"configured", []string{"grammar", "grammar"}, nil, true, "grammar,grammar"},
{"empty override", []string{"grammar"}, []string{}, true, ""},
} {
t.Run(test.name, func(t *testing.T) {
runner := auditaTestRunner(t, SubprocessRunnerConfig{Modules: test.configured})
req := auditaTestRequest(t)
req.Modules = test.requested
runner.run = func(_ context.Context, processReq subprocess.RunRequest) (subprocess.RunResult, error) {
index := -1
for i, arg := range processReq.Args {
if arg == "--modules" {
index = i
}
if strings.HasPrefix(arg, "--report") || arg == "--debug" || arg == "--work-dir" {
t.Fatalf("retired or unwanted flag %q", arg)
}
}
if (index >= 0) != test.wantFlag || (index >= 0 && processReq.Args[index+1] != test.wantValue) {
t.Fatalf("args = %#v", processReq.Args)
}
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
return subprocess.RunResult{}, nil
}
result, err := runner.Run(context.Background(), req)
if err != nil || result.Reports != (ReportBundle{}) {
t.Fatalf("result = %#v, error = %v", result, err)
}
})
}
}
func TestSubprocessRunnerRejectsMissingOrMalformedTranscript(t *testing.T) {
for _, output := range []string{"", `{"segments":"wrong"}`} {
t.Run(output, func(t *testing.T) {
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
req := auditaTestRequest(t)
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
if output != "" {
writeAuditaTestFile(t, req.OutputProcessedPath, output)
}
return subprocess.RunResult{}, nil
}
if _, err := runner.Run(context.Background(), req); err == nil {
t.Fatal("missing or malformed transcript was accepted")
}
})
}
}
func TestSubprocessRunnerPreservesFailureAndAvailableReports(t *testing.T) {
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
req := auditaTestRequest(t)
processFailure := errors.New("audita exited after writing output")
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
writeAuditaTestFile(t, filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml"), "status: failed\n")
return subprocess.RunResult{ExitCode: 7}, processFailure
}
result, err := runner.Run(context.Background(), req)
if !errors.Is(err, processFailure) || !strings.Contains(err.Error(), filepath.Dir(req.OutputProcessedPath)) {
t.Fatalf("error = %v", err)
}
if result.Reports.ErrorsPath == "" || result.Reports.RecoveriesPath != "" || result.Reports.OverviewPath != "" {
t.Fatalf("reports = %#v", result.Reports)
}
}
func TestSubprocessRunnerDoesNotMaskProcessFailureWithReportError(t *testing.T) {
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
req := auditaTestRequest(t)
processFailure := errors.New("audita process failed")
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
path := filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml")
if err := os.WriteFile(path, make([]byte, MaxReportOutputBytes+1), 0o600); err != nil {
t.Fatal(err)
}
return subprocess.RunResult{ExitCode: 7}, processFailure
}
if _, err := runner.Run(context.Background(), req); !errors.Is(err, processFailure) {
t.Fatalf("error = %v, want original process error", err)
}
}
func TestSubprocessRunnerBoundsAdjacentReports(t *testing.T) {
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
req := auditaTestRequest(t)
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
path := filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml")
if err := os.WriteFile(path, make([]byte, MaxReportOutputBytes+1), 0o600); err != nil {
t.Fatal(err)
}
return subprocess.RunResult{}, nil
}
if _, err := runner.Run(context.Background(), req); err == nil || !strings.Contains(err.Error(), "errors.yml") || !strings.Contains(err.Error(), "16777216") {
t.Fatalf("error = %v, want named report limit", err)
}
}
func TestSubprocessRunnerRedactsCustomCredentialStreams(t *testing.T) {
const first = "NARRATIO_AUDITA_STREAM_ONE"
const second = "NARRATIO_AUDITA_STREAM_TWO"
for _, fail := range []bool{false, true} {
t.Run(map[bool]string{false: "success", true: "failure"}[fail], func(t *testing.T) {
firstSecret := auditaTestSecret(t, first, "audita-first-secret")
secondSecret := auditaTestSecret(t, second, "audita-second-secret")
req := auditaTestRequest(t)
body := "#!/bin/sh\nprintf '%s\\n' \"$" + first + "\"\nprintf '%s\\n' \"$" + second + "\" >&2\n"
if fail {
body += "exit 7\n"
} else {
body += "printf '{\"segments\":[]}' > \"" + req.OutputProcessedPath + "\"\n"
}
binary := filepath.Join(t.TempDir(), "audita-helper")
writeAuditaTestFile(t, binary, body)
if err := os.Chmod(binary, 0o700); err != nil {
t.Fatal(err)
}
runner := auditaTestRunner(t, SubprocessRunnerConfig{Binary: binary, Credentials: []configkit.Secret{firstSecret, secondSecret}})
_, err := runner.Run(context.Background(), req)
if (err != nil) != fail {
t.Fatalf("error = %v, fail = %v", err, fail)
}
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
data, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatal(readErr)
}
if strings.Contains(string(data), "audita-first-secret") || strings.Contains(string(data), "audita-second-secret") || !strings.Contains(string(data), "<redacted>") {
t.Fatalf("unredacted stream %q: %s", path, data)
}
}
if fail && (strings.Contains(err.Error(), "audita-first-secret") || strings.Contains(err.Error(), "audita-second-secret")) {
t.Fatalf("unredacted failure: %v", err)
}
})
}
return d
}

View File

@@ -1,6 +1,10 @@
package notarius
import "context"
import (
"context"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
// FakeRunner is a configurable in-memory runner for stage tests.
type FakeRunner struct {
@@ -15,6 +19,11 @@ func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error)
return RunResult{}, err
}
copyRequest := req
copyRequest.Credentials = append([]configkit.Secret(nil), req.Credentials...)
if req.ConfigProfile != nil {
profile := *req.ConfigProfile
copyRequest.ConfigProfile = &profile
}
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
f.Requests = append(f.Requests, copyRequest)
if f.Err != nil {

View File

@@ -4,6 +4,8 @@ package notarius
import (
"context"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
const ReceiptSchemaVersion = "notarius.run-result.v2"
@@ -22,7 +24,10 @@ type ReferenceBinding struct {
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
type RunRequest struct {
Binary string
Binary string
// Credentials are already-resolved environment references selected for Notarius.
Credentials []configkit.Secret
ConfigProfile *string
ConfigPath string
PipelineID string
InputPath string
@@ -31,6 +36,8 @@ type RunRequest struct {
ReceiptPath string
LogPath string
Timeout time.Duration
Resume bool
ChunkCache string
References []ReferenceBinding
}

View File

@@ -52,6 +52,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
if err != nil {
return RunResult{}, err
}
envOverrides, sensitiveEnvNames, err := subprocess.CredentialEnvironment(req.Credentials)
if err != nil {
return RunResult{}, fmt.Errorf("notarius credentials: %w", err)
}
args := []string{
"run", req.PipelineID,
@@ -59,18 +63,29 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
"--input", req.InputPath,
"--output-dir", req.OutputRoot,
}
if req.Resume {
args = append(args, "--resume")
}
if req.ChunkCache != "" {
args = append(args, "--chunk_cache", req.ChunkCache)
}
for _, reference := range references {
args = append(args, "--reference", reference.Selector+"="+reference.Path)
}
if req.ConfigProfile != nil {
args = append(args, "--config-profile", *req.ConfigProfile)
}
args = append(args, "--json")
processResult, err := r.run(ctx, subprocess.RunRequest{
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDirectory,
Timeout: req.Timeout,
DiagnosticOwner: "notarius",
StdoutLogPath: req.ReceiptPath,
StderrLogPath: req.LogPath,
Executable: req.Binary,
Args: args,
WorkingDir: req.WorkingDirectory,
Timeout: req.Timeout,
DiagnosticOwner: "notarius",
StdoutLogPath: req.ReceiptPath,
StderrLogPath: req.LogPath,
EnvOverrides: envOverrides,
SensitiveEnvNames: sensitiveEnvNames,
})
baseResult := RunResult{
ReceiptPath: req.ReceiptPath,
@@ -143,6 +158,9 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
if req.Timeout <= 0 {
return nil, fmt.Errorf("notarius timeout must be positive")
}
if !validChunkCacheMode(req.ChunkCache) {
return nil, fmt.Errorf("notarius chunk cache mode must be empty, auto, bypass, or refresh")
}
for label, path := range map[string]string{
"config": req.ConfigPath,
"input": req.InputPath,
@@ -201,6 +219,15 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
return references, nil
}
func validChunkCacheMode(mode string) bool {
switch mode {
case "", "auto", "bypass", "refresh":
return true
default:
return false
}
}
type receiptDocument struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
@@ -248,7 +275,7 @@ func validateValidationSummaries(documents []validationSummaryDocument) ([]Valid
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
}
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) {
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount < 0 || !validTerminalAction(document.TerminalAction) {
return nil, fmt.Errorf("notarius validation summary is missing required fields")
}
summaries = append(summaries, ValidationSummary{

View File

@@ -11,6 +11,7 @@ import (
"testing"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
)
@@ -76,6 +77,178 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
}
}
func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
tests := []struct {
name string
resume bool
chunkCache string
want []string
}{
{name: "resume", resume: true, want: []string{"--resume"}},
{name: "auto", chunkCache: "auto", want: []string{"--chunk_cache", "auto"}},
{name: "bypass", chunkCache: "bypass", want: []string{"--chunk_cache", "bypass"}},
{name: "refresh", chunkCache: "refresh", want: []string{"--chunk_cache", "refresh"}},
{name: "resume and mode", resume: true, chunkCache: "auto", want: []string{"--resume", "--chunk_cache", "auto"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
req.Resume = test.resume
req.ChunkCache = test.chunkCache
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
want := []string{
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot,
}
want = append(want, test.want...)
want = append(want, "--json")
if !reflect.DeepEqual(captured.Args, want) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, want)
}
})
}
}
func TestSubprocessRunnerMapsConfigProfileToArgumentPair(t *testing.T) {
tests := []struct {
name string
supplied bool
value string
}{
{name: "omitted"},
{name: "empty", supplied: true},
{name: "default", supplied: true, value: "default"},
{name: "named", supplied: true, value: "inexpensive"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
if test.supplied {
req.ConfigProfile = &test.value
}
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
flagIndex := -1
for index, arg := range captured.Args {
if arg == "--config-profile" {
flagIndex = index
break
}
}
if !test.supplied {
if flagIndex >= 0 {
t.Fatalf("subprocess args unexpectedly contain config profile: %#v", captured.Args)
}
return
}
if flagIndex < 0 || flagIndex+2 >= len(captured.Args) || captured.Args[flagIndex+1] != test.value || captured.Args[flagIndex+2] != "--json" {
t.Fatalf("config profile argument boundary = %#v, want flag, exact value, then --json", captured.Args)
}
})
}
}
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
for _, mode := range []string{"AUTO", "invalid", " auto "} {
t.Run(mode, func(t *testing.T) {
req := validRunRequest(t)
req.ChunkCache = mode
started := false
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
started = true
return sharedsubprocess.RunResult{}, nil
}}
_, err := runner.Run(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "chunk cache mode") {
t.Fatalf("Run() error = %v, want chunk cache validation failure", err)
}
if started {
t.Fatal("subprocess started with invalid chunk cache mode")
}
})
}
}
func TestSubprocessRunnerForwardsLoadedCredentialWithoutRereadingEnvironment(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_LLM_KEY"
const otherName = "NARRATIO_NOTARIUS_CUSTOM_OTHER_KEY"
const loadedValue = "loaded-notarius-credential"
const otherValue = "loaded-notarius-other-credential"
credential := loadTestCredential(t, envName, loadedValue)
otherCredential := loadTestCredential(t, otherName, otherValue)
t.Setenv(envName, "changed-parent-credential")
t.Setenv(otherName, "changed-parent-other-credential")
req := validRunRequest(t)
req.Credentials = []configkit.Secret{*credential, *otherCredential}
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(captured.EnvOverrides) != 2 || captured.EnvOverrides[envName] != loadedValue || captured.EnvOverrides[otherName] != otherValue {
t.Fatal("subprocess did not receive exactly the loaded credential overrides")
}
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{envName, otherName}) {
t.Fatalf("sensitive environment names = %#v, want configured reference name", captured.SensitiveEnvNames)
}
wantArgs := []string{
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
"--output-dir", req.OutputRoot, "--json",
}
if !reflect.DeepEqual(captured.Args, wantArgs) {
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
}
}
func TestSubprocessRunnerRejectsInvalidProgrammaticCredentialBeforeLaunch(t *testing.T) {
for _, test := range []struct {
name string
reference configkit.Secret
}{
{name: "unresolved", reference: configkit.SecretEnv("NARRATIO_NOTARIUS_UNRESOLVED_KEY")},
{name: "empty", reference: configkit.SecretEnv("")},
} {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
req.Credentials = []configkit.Secret{test.reference}
started := false
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
started = true
return sharedsubprocess.RunResult{}, nil
}}
_, err := runner.Run(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "resolved and nonempty") {
t.Fatalf("Run() error = %v, want credential validation failure", err)
}
if started {
t.Fatal("subprocess started with an invalid credential")
}
})
}
}
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
req := validRunRequest(t)
referenceRoot := t.TempDir()
@@ -172,16 +345,25 @@ func TestSubprocessRunnerRejectsInvalidReferencesBeforeLaunch(t *testing.T) {
}
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_KEY"
const otherName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_OTHER_KEY"
const canary = "notarius-captured-stream-canary"
const otherCanary = "notarius-captured-stream-other-canary"
const unrelated = "NARRATIO_NOTARIUS_UNRELATED_PARENT_KEY"
req := validRunRequest(t)
req.Credentials = []configkit.Secret{*loadTestCredential(t, envName, canary), *loadTestCredential(t, otherName, otherCanary)}
writeValidBundleAndReceipt(t, req, false)
receiptFixture := req.ReceiptPath + ".fixture"
data, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
var receipt map[string]any
if err := json.Unmarshal(data, &receipt); err != nil {
t.Fatalf("decode receipt fixture: %v", err)
}
receipt["credential_echo"] = canary + " " + otherCanary
writeJSONFile(t, receiptFixture, receipt)
if err := os.Remove(req.ReceiptPath); err != nil {
t.Fatalf("Remove(receipt) error = %v", err)
}
@@ -193,27 +375,67 @@ func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T)
script := writeShellScript(t, `#!/bin/sh
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
printf 'diagnostic stream\n' >&2
if [ "$`+envName+`" != '`+canary+`' ]; then
printf 'selected credential mismatch\n' >&2
exit 8
fi
if [ "$`+otherName+`" != '`+otherCanary+`' ]; then
printf 'other selected credential mismatch\n' >&2
exit 10
fi
if [ -n "$`+unrelated+`" ]; then
printf 'unrelated credential inherited\n' >&2
exit 9
fi
printf '%s\n' "$`+envName+`" >&2
printf '%s\n' "$`+otherName+`" >&2
cat "$NOTARIUS_RECEIPT_FIXTURE"
`)
req.Binary = script
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
t.Setenv(unrelated, "unrelated-parent-credential")
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
assertTextFile(t, req.LogPath, "diagnostic stream\n")
assertRedactedCredentialFile(t, req.LogPath, canary)
assertRedactedCredentialFile(t, req.LogPath, otherCanary)
receiptBytes, err := os.ReadFile(req.ReceiptPath)
if err != nil {
t.Fatalf("ReadFile(receipt) error = %v", err)
}
if strings.Contains(string(receiptBytes), "diagnostic stream") {
t.Fatal("receipt contains stderr output")
assertRedactedCredentialBytes(t, receiptBytes, canary)
assertRedactedCredentialBytes(t, receiptBytes, otherCanary)
}
func TestSubprocessRunnerRedactsCredentialOnProcessFailure(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_KEY"
const otherName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_OTHER_KEY"
const canary = "notarius-failure-stream-canary"
const otherCanary = "notarius-failure-stream-other-canary"
req := validRunRequest(t)
req.Credentials = []configkit.Secret{*loadTestCredential(t, envName, canary), *loadTestCredential(t, otherName, otherCanary)}
req.Binary = writeShellScript(t, `#!/bin/sh
printf '%s\n' "$`+envName+`"
printf '%s\n' "$`+envName+`" >&2
printf '%s\n' "$`+otherName+`" >&2
exit 7
`)
_, err := NewSubprocessRunner().Run(context.Background(), req)
if err == nil {
t.Fatal("Run() error = nil, want process failure")
}
if strings.Contains(err.Error(), canary) || strings.Contains(err.Error(), otherCanary) || !strings.Contains(err.Error(), "<redacted>") {
t.Fatal("process failure did not return redacted diagnostics")
}
assertRedactedCredentialFile(t, req.ReceiptPath, canary)
assertRedactedCredentialFile(t, req.ReceiptPath, otherCanary)
assertRedactedCredentialFile(t, req.LogPath, canary)
assertRedactedCredentialFile(t, req.LogPath, otherCanary)
}
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
@@ -231,6 +453,10 @@ func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
if test.name == "nonzero" {
profile := "missing-profile"
req.ConfigProfile = &profile
}
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
req.Timeout = test.timeout
ctx := context.Background()
@@ -527,7 +753,8 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin
}
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
req := RunRequest{PipelineID: "pipeline", References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
profile := "testing"
req := RunRequest{PipelineID: "pipeline", ConfigProfile: &profile, References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
want := RunResult{BundleRoot: "/bundle"}
fake := &FakeRunner{Result: want}
got, err := fake.Run(context.Background(), req)
@@ -538,6 +765,10 @@ func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
if fake.Requests[0].References[0].Path != "/references/party.json" {
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
}
profile = "changed"
if *fake.Requests[0].ConfigProfile != "testing" {
t.Fatalf("fake retained aliased config profile: %q", *fake.Requests[0].ConfigProfile)
}
wantErr := errors.New("configured failure")
fake.Err = wantErr
@@ -579,6 +810,56 @@ func validRunRequest(t *testing.T) RunRequest {
}
}
func loadTestCredential(t *testing.T, envName, value string) *configkit.Secret {
t.Helper()
t.Setenv(envName, value)
type testConfig struct {
Credential *configkit.Secret `yaml:"credential"`
}
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+envName+"\n")), configkit.Options[testConfig]{})
if err != nil {
t.Fatalf("load test credential: %v", err)
}
return result.Config.Credential
}
func assertRedactedCredentialFile(t *testing.T, path, canary string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
assertRedactedCredentialBytes(t, data, canary)
}
func assertRedactedCredentialBytes(t *testing.T, data []byte, canary string) {
t.Helper()
text := string(data)
if strings.Contains(text, canary) || !strings.Contains(text, "<redacted>") {
t.Fatal("captured stream did not contain the required redaction marker")
}
}
func TestValidateValidationSummariesAcceptsReusedProducerWithoutAttempts(t *testing.T) {
zero := 0
summaries, err := validateValidationSummaries([]validationSummaryDocument{{
Stage: "chunk", Status: "complete", ProducerAttemptCount: &zero, TerminalAction: "accepted",
}})
if err != nil {
t.Fatalf("validateValidationSummaries() error = %v", err)
}
if len(summaries) != 1 || summaries[0].ProducerAttemptCount != 0 {
t.Fatalf("summaries = %#v, want one reused producer with zero attempts", summaries)
}
negative := -1
if _, err := validateValidationSummaries([]validationSummaryDocument{{
Stage: "chunk", Status: "complete", ProducerAttemptCount: &negative, TerminalAction: "accepted",
}}); err == nil {
t.Fatal("validateValidationSummaries() negative producer attempts error = nil")
}
}
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
t.Helper()
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")

View File

@@ -6,16 +6,12 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
}
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
}
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
}
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
}
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json")

View File

@@ -6,16 +6,12 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "success")
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
}
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
}
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
}
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
}
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
}
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
}
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
}
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
}
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
}
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
}
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
}
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
}
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
}
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
}
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
}
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
}
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
}
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
}
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
recordPath := filepath.Join(t.TempDir(), "record.json")
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
}
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "fail")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
}
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
}
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("helper wrapper script uses /bin/sh")
}
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))

View File

@@ -37,8 +37,12 @@ type s3ClientOptions struct {
Region string
Endpoint string
ForcePathStyle bool
AccessKeyID string
SecretKey string
Credentials *s3StaticCredentials
}
type s3StaticCredentials struct {
AccessKeyID string
SecretAccessKey string
}
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
@@ -46,11 +50,11 @@ var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error)
if strings.TrimSpace(opts.Region) != "" {
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
}
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
if opts.Credentials != nil {
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
strings.TrimSpace(opts.AccessKeyID),
strings.TrimSpace(opts.SecretKey),
opts.Credentials.AccessKeyID,
opts.Credentials.SecretAccessKey,
"",
),
))
@@ -75,14 +79,24 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
if bucket == "" {
return nil, fmt.Errorf("storage.s3.bucket is required")
}
if (cfg.AccessKeyID == nil) != (cfg.SecretAccessKey == nil) {
return nil, fmt.Errorf("storage.s3 access key ID and secret access key references must be configured together")
}
if cfg.AccessKeyID != nil && (!cfg.AccessKeyID.IsResolved() || !cfg.SecretAccessKey.IsResolved()) {
return nil, fmt.Errorf("storage.s3 configured credential references must be resolved")
}
client, err := newS3Client(ctx, s3ClientOptions{
clientOptions := s3ClientOptions{
Region: cfg.Region,
Endpoint: cfg.Endpoint,
ForcePathStyle: cfg.ForcePathStyle,
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
})
}
if cfg.AccessKeyID != nil && cfg.SecretAccessKey != nil {
clientOptions.Credentials = &s3StaticCredentials{
AccessKeyID: cfg.AccessKeyID.Value(), SecretAccessKey: cfg.SecretAccessKey.Value(),
}
}
client, err := newS3Client(ctx, clientOptions)
if err != nil {
return nil, fmt.Errorf("build s3 client: %w", err)
}
@@ -93,26 +107,6 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
}, nil
}
func s3CredentialFromEnv(envVarName string) string {
name := strings.TrimSpace(envVarName)
if name == "" {
return ""
}
value, ok := os.LookupEnv(name)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
func orDefaultEnvName(name, fallback string) string {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
return fallback
}
return trimmed
}
// List returns objects under prefix.
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
normalizedPrefix := normalizeObjectKey(prefix)

View File

@@ -14,6 +14,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
@@ -319,8 +320,10 @@ func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
accessKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_ACCESS", "id-123")
secretKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_SECRET", "secret-abc")
t.Setenv("NARRATIO_S3_TEST_ACCESS", "changed-id")
t.Setenv("NARRATIO_S3_TEST_SECRET", "changed-secret")
var got s3ClientOptions
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
@@ -329,10 +332,12 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
}
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
Endpoint: "http://localhost:9000",
ForcePathStyle: true,
Bucket: "my-archive",
Region: "us-east-1",
Endpoint: "http://localhost:9000",
ForcePathStyle: true,
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
@@ -343,7 +348,7 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
}
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
if got.Credentials == nil || got.Credentials.AccessKeyID != "id-123" || got.Credentials.SecretAccessKey != "secret-abc" {
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
}
}
@@ -355,7 +360,21 @@ func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
}
}
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
func TestNewS3BackendFromConfigRejectsIncompleteOrUnresolvedCredentials(t *testing.T) {
unresolved := configkit.SecretEnv("NARRATIO_UNRESOLVED_S3_KEY")
for name, cfg := range map[string]config.StorageS3Config{
"partial": {Bucket: "my-archive", AccessKeyID: &unresolved},
"unresolved": {Bucket: "my-archive", AccessKeyID: &unresolved, SecretAccessKey: &unresolved},
} {
t.Run(name, func(t *testing.T) {
if _, err := NewS3BackendFromConfig(context.Background(), cfg); err == nil {
t.Fatal("NewS3BackendFromConfig() succeeded, want credential error")
}
})
}
}
func TestNewS3BackendFromConfigUsesDefaultProviderWithoutCredentialReferences(t *testing.T) {
original := newS3Client
t.Cleanup(func() { newS3Client = original })
@@ -366,19 +385,30 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
}
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
Bucket: "my-archive",
Region: "us-east-1",
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
SecretKeyEnv: "MISSING_SECRET_KEY",
Bucket: "my-archive",
Region: "us-east-1",
})
if err != nil {
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
}
if got.AccessKeyID != "" || got.SecretKey != "" {
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
if got.Credentials != nil {
t.Fatalf("client options credentials = %#v, want default provider chain", got.Credentials)
}
}
func resolvedTestSecret(t *testing.T, name, value string) *configkit.Secret {
t.Helper()
t.Setenv(name, value)
type document struct {
Secret *configkit.Secret `yaml:"secret"`
}
result, err := configkit.Load(configkit.Bytes("test.yml", []byte("secret: "+name+"\n")), configkit.Options[document]{})
if err != nil {
t.Fatalf("resolve test secret: %v", err)
}
return result.Config.Secret
}
func strPtr(v string) *string { return &v }
func int64Ptr(v int64) *int64 { return &v }
func boolPtr(v bool) *bool { return &v }

View File

@@ -0,0 +1,29 @@
package subprocess
import (
"fmt"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
// CredentialEnvironment returns captured secret values and names for a child
// invocation. References must already have been resolved by ConfigKit.
func CredentialEnvironment(credentials []configkit.Secret) (map[string]string, []string, error) {
if len(credentials) == 0 {
return nil, nil, nil
}
overrides := make(map[string]string, len(credentials))
names := make([]string, 0, len(credentials))
for _, credential := range credentials {
name := credential.EnvName()
value := credential.Value()
if !credential.IsResolved() || value == "" {
return nil, nil, fmt.Errorf("credential %q must be resolved and nonempty", name)
}
if _, exists := overrides[name]; !exists {
names = append(names, name)
overrides[name] = value
}
}
return overrides, names, nil
}

View File

@@ -0,0 +1,51 @@
package subprocess
import (
"reflect"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestCredentialEnvironmentUsesCapturedReferences(t *testing.T) {
t.Setenv("CUSTOM_ALPHA_KEY", "alpha-value")
t.Setenv("SECOND_PROVIDER_TOKEN", "second-value")
t.Setenv("UNSELECTED_SECRET", "unselected-value")
type configuration struct {
Credentials []configkit.Secret `yaml:"credentials"`
}
loaded, err := configkit.Load[configuration](configkit.Bytes("credentials.yml", []byte("credentials: [CUSTOM_ALPHA_KEY, SECOND_PROVIDER_TOKEN, CUSTOM_ALPHA_KEY]\n")))
if err != nil {
t.Fatal(err)
}
t.Setenv("CUSTOM_ALPHA_KEY", "later-value")
overrides, names, err := CredentialEnvironment(loaded.Config.Credentials)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(overrides, map[string]string{"CUSTOM_ALPHA_KEY": "alpha-value", "SECOND_PROVIDER_TOKEN": "second-value"}) ||
!reflect.DeepEqual(names, []string{"CUSTOM_ALPHA_KEY", "SECOND_PROVIDER_TOKEN"}) {
t.Fatalf("credential selection = %#v, %#v", overrides, names)
}
overrides["CUSTOM_ALPHA_KEY"] = "modified"
names[0] = "modified"
again, againNames, err := CredentialEnvironment(loaded.Config.Credentials)
if err != nil || again["CUSTOM_ALPHA_KEY"] != "alpha-value" || againNames[0] != "CUSTOM_ALPHA_KEY" {
t.Fatalf("credential conversion retained mutable result: %#v, %#v, %v", again, againNames, err)
}
}
func TestCredentialEnvironmentHandlesEmptyAndUnresolvedReferences(t *testing.T) {
for _, credentials := range [][]configkit.Secret{nil, {}} {
overrides, names, err := CredentialEnvironment(credentials)
if err != nil || len(overrides) != 0 || len(names) != 0 {
t.Fatalf("empty credentials = %#v, %#v, %v", overrides, names, err)
}
}
for _, credentials := range [][]configkit.Secret{{configkit.SecretEnv("UNRESOLVED_KEY")}, {{}}} {
overrides, names, err := CredentialEnvironment(credentials)
if err == nil || overrides != nil || names != nil {
t.Fatalf("unresolved credentials = %#v, %#v, %v", overrides, names, err)
}
}
}

View File

@@ -1,4 +1,4 @@
//go:build linux || darwin || windows
//go:build linux || darwin
package subprocess

View File

@@ -1,10 +1,9 @@
//go:build !linux && !darwin && !windows
//go:build !linux && !darwin
package subprocess
import (
"fmt"
"runtime"
)
func newOwnedProcessTree() (ownedProcessTree, error) {

View File

@@ -1,145 +0,0 @@
//go:build windows
package subprocess
import (
"errors"
"fmt"
"os/exec"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
type windowsProcessTree struct {
job windows.Handle
}
func newOwnedProcessTree() (ownedProcessTree, error) {
return &windowsProcessTree{}, nil
}
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
job, err := windows.CreateJobObject(nil, nil)
if err != nil {
return fmt.Errorf("create job object: %w", err)
}
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
_ = windows.CloseHandle(job)
return fmt.Errorf("configure job object: %w", err)
}
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
if err := cmd.Start(); err != nil {
_ = windows.CloseHandle(job)
return err
}
var assignErr error
withHandleErr := cmd.Process.WithHandle(func(handle uintptr) {
assignErr = windows.AssignProcessToJobObject(job, windows.Handle(handle))
})
if withHandleErr != nil {
return failWindowsProcessStart(cmd, job, fmt.Errorf("acquire stable process handle for job assignment: %w", withHandleErr))
}
if assignErr != nil {
return failWindowsProcessStart(cmd, job, fmt.Errorf("assign process to job object: %w", assignErr))
}
if err := resumeInitialThread(uint32(cmd.Process.Pid)); err != nil {
return failWindowsProcessStart(cmd, job, fmt.Errorf("resume initial process thread: %w", err))
}
tree.job = job
return nil
}
func failWindowsProcessStart(cmd *exec.Cmd, job windows.Handle, setupErr error) error {
killErr := cmd.Process.Kill()
waitErr := cmd.Wait()
var exitErr *exec.ExitError
if killErr == nil && errors.As(waitErr, &exitErr) {
// A killed suspended child normally reports a non-zero exit from Wait;
// successful reaping is not an additional setup failure.
waitErr = nil
}
return joinErrors(
setupErr,
wrapWindowsProcessCleanupError("kill suspended child", killErr),
wrapWindowsProcessCleanupError("reap suspended child", waitErr),
wrapWindowsProcessCleanupError("close unowned job object", windows.CloseHandle(job)),
)
}
func wrapWindowsProcessCleanupError(operation string, err error) error {
if err == nil {
return nil
}
return fmt.Errorf("%s: %w", operation, err)
}
func resumeInitialThread(processID uint32) error {
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
if err != nil {
return fmt.Errorf("snapshot initial thread: %w", err)
}
defer func() { _ = windows.CloseHandle(snapshot) }()
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
if err := windows.Thread32First(snapshot, &entry); err != nil {
return fmt.Errorf("find initial thread: %w", err)
}
for {
if entry.OwnerProcessID != processID {
// Keep enumerating until the suspended process's only initial thread
// is found.
} else {
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
if openErr != nil {
return fmt.Errorf("open initial thread: %w", openErr)
}
defer func() { _ = windows.CloseHandle(thread) }()
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
return fmt.Errorf("resume initial thread: %w", resumeErr)
}
return nil
}
if err := windows.Thread32Next(snapshot, &entry); err != nil {
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
break
}
return fmt.Errorf("find initial thread: %w", err)
}
}
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
}
func (tree *windowsProcessTree) TerminateGracefully() error {
// Windows jobs have no portable graceful signal. Terminating the owned job
// is the safe fallback and prevents a descendant from escaping cleanup.
return tree.terminate()
}
func (tree *windowsProcessTree) TerminateForcefully() error {
return tree.terminate()
}
func (tree *windowsProcessTree) Dispose() error {
if tree.job == 0 {
return nil
}
err := windows.CloseHandle(tree.job)
tree.job = 0
return err
}
func (tree *windowsProcessTree) terminate() error {
if tree.job == 0 {
return nil
}
return windows.TerminateJobObject(tree.job, 1)
}

View File

@@ -8,7 +8,6 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"strings"
"syscall"
@@ -137,9 +136,6 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
}
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
}
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable() error = %v", err)

View File

@@ -6,14 +6,11 @@ import (
"errors"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
@@ -21,7 +18,6 @@ import (
const (
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
whisperXUploadBufferSize = 32 * 1024
)
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
@@ -192,21 +188,32 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
}
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
defer upload.Close()
upload, err := newMultipartBodyFactory(ctx, audioPath, c.language, c.openAudio)
if err != nil {
return 0, nil, fmt.Errorf("prepare whisperx request body: %w", err)
}
defer func() { _ = upload.Close() }()
body, err := upload.NewBody()
if err != nil {
return 0, nil, fmt.Errorf("open whisperx request body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), body)
if err != nil {
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
}
req.Header.Set("Content-Type", upload.contentType)
req.Header.Set("Expect", "100-continue")
req.GetBody = upload.NewBody
resp, err := c.httpClient.Do(req)
if err != nil {
_ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil {
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
if sourceErr := upload.SourceError(true); sourceErr != nil {
return 0, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
}
if ctxErr := ctx.Err(); ctxErr != nil {
return 0, nil, fmt.Errorf("perform whisperx request: %w", ctxErr)
}
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
}
@@ -214,8 +221,8 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
_ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
if sourceErr := upload.SourceError(false); sourceErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
}
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
@@ -223,13 +230,19 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
}
if err := upload.Wait(); err != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
data, responseErr := readWhisperXResponse(resp.Body, c.maxResponseBytes)
_ = upload.Close()
if sourceErr := upload.SourceError(false); sourceErr != nil {
if responseErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", errors.Join(sourceErr, responseErr))
}
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
}
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
if err != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
if err := ctx.Err(); err != nil {
return resp.StatusCode, nil, fmt.Errorf("complete whisperx request: %w", err)
}
if responseErr != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", responseErr)
}
return resp.StatusCode, data, nil
}
@@ -286,161 +299,6 @@ func isHTTPURLScheme(scheme string) bool {
}
}
type multipartUpload struct {
reader *io.PipeReader
writer *io.PipeWriter
contentType string
done chan struct{}
mu sync.Mutex
audio io.Closer
err error
aborted bool
}
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
reader, writer := io.Pipe()
multipartWriter := multipart.NewWriter(writer)
upload := &multipartUpload{
reader: reader,
writer: writer,
contentType: multipartWriter.FormDataContentType(),
done: make(chan struct{}),
}
go func() {
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
if err != nil {
_ = writer.CloseWithError(err)
} else {
_ = writer.Close()
}
upload.mu.Lock()
upload.err = err
upload.audio = nil
upload.mu.Unlock()
close(upload.done)
}()
go func() {
select {
case <-ctx.Done():
upload.abort()
case <-upload.done:
}
}()
return upload
}
func (u *multipartUpload) Read(p []byte) (int, error) {
return u.reader.Read(p)
}
func (u *multipartUpload) Close() error {
u.abort()
return nil
}
func (u *multipartUpload) Wait() error {
<-u.done
u.mu.Lock()
defer u.mu.Unlock()
return u.err
}
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
if err != nil {
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
}
audioFile, err := openAudio(audioPath)
if err != nil {
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
}
u.setAudio(audioFile)
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
closeErr := audioFile.Close()
u.clearAudio(audioFile)
if copyErr != nil {
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
}
if closeErr != nil {
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
}
if err := writer.WriteField("language", language); err != nil {
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
}
if err := writer.Close(); err != nil {
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
}
return nil
}
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
if ctx.Err() != nil {
return ctx.Err()
}
u.mu.Lock()
aborted := u.aborted
u.mu.Unlock()
if aborted {
return nil
}
return err
}
func (u *multipartUpload) setAudio(audio io.Closer) {
u.mu.Lock()
u.audio = audio
aborted := u.aborted
u.mu.Unlock()
if aborted {
_ = audio.Close()
}
}
func (u *multipartUpload) clearAudio(audio io.Closer) {
u.mu.Lock()
if u.audio == audio {
u.audio = nil
}
u.mu.Unlock()
}
func (u *multipartUpload) abort() {
u.mu.Lock()
if u.aborted {
u.mu.Unlock()
return
}
u.aborted = true
audio := u.audio
u.mu.Unlock()
_ = u.reader.Close()
if audio != nil {
_ = audio.Close()
}
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r *contextReader) Read(p []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.reader.Read(p)
}
}
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("path is required")

View File

@@ -0,0 +1,261 @@
package whisperx
import (
"context"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestHTTPClientTransportReplaysAbandonedMultipartBody(t *testing.T) {
var opens atomic.Int32
var closes atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
partial := make([]byte, 17)
if _, err := io.ReadFull(request.Body, partial); err != nil {
return nil, err
}
if err := request.Body.Close(); err != nil {
return nil, err
}
if request.GetBody == nil {
return nil, errors.New("request GetBody is nil")
}
replay, err := request.GetBody()
if err != nil {
return nil, err
}
replayedBytes, err := io.ReadAll(replay)
if err != nil {
_ = replay.Close()
return nil, err
}
if err := replay.Close(); err != nil {
return nil, err
}
parts := parseMultipartParts(t, request.Header.Get("Content-Type"), replayedBytes)
if len(parts) != 2 || parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
t.Fatalf("replayed file part = %#v, want exact audio.flac payload", parts)
}
if parts[1].name != "language" || string(parts[1].data) != "en" {
t.Fatalf("replayed language part = %#v, want language=en", parts[1])
}
return whisperXJSONResponse(request), nil
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
opens.Add(1)
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
result, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
if result.Attempts != 1 {
t.Fatalf("application attempts = %d, want 1", result.Attempts)
}
verifyJSONFile(t, outPath)
if got := opens.Load(); got != 2 {
t.Fatalf("audio opens = %d, want original and replay", got)
}
if got := closes.Load(); got != 2 {
t.Fatalf("audio closes = %d, want all body handles disposed", got)
}
}
func TestHTTPClientTransportReturnsReplayOpenFailure(t *testing.T) {
replayErr := errors.New("replay open failed")
var opens atomic.Int32
var closes atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
_, err := request.GetBody()
return nil, err
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 2 {
return nil, replayErr
}
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
result, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if !errors.Is(err, replayErr) {
t.Fatalf("Transcribe() error = %v, want replay open failure", err)
}
if result.Attempts != 1 {
t.Fatalf("application attempts = %d, want 1", result.Attempts)
}
if got := closes.Load(); got != 1 {
t.Fatalf("audio closes = %d, want original disposed", got)
}
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("output should not exist, stat error = %v", statErr)
}
}
func TestHTTPClientTransportCannotSwallowReplayReadFailure(t *testing.T) {
replayErr := errors.New("replay read failed")
var opens atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
replay, err := request.GetBody()
if err != nil {
return nil, err
}
_, _ = io.ReadAll(replay)
_ = replay.Close()
return whisperXJSONResponse(request), nil
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 2 {
return &failingReadCloser{first: []byte("partial"), err: replayErr}, nil
}
return io.NopCloser(strings.NewReader("audio-data")), nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
_, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if !errors.Is(err, replayErr) {
t.Fatalf("Transcribe() error = %v, want replay read failure", err)
}
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("output should not exist, stat error = %v", statErr)
}
}
func TestHTTPClientTransportEarlyResponseDisposesOriginalBody(t *testing.T) {
var closes atomic.Int32
client := newReplayTestHTTPClient(t, roundTripFunc(func(request *http.Request) (*http.Response, error) {
return whisperXJSONResponse(request), nil
}))
client.openAudio = func(string) (io.ReadCloser, error) {
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
_, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
})
if err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
if got := closes.Load(); got != 1 {
t.Fatalf("audio closes = %d, want early-response body disposed", got)
}
}
func TestHTTPClientParentCancellationDuringReplayClosesEveryBody(t *testing.T) {
var opens atomic.Int32
var originalCloses atomic.Int32
replaySource := newSignalingBlockedReadCloser()
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
replay, err := request.GetBody()
if err != nil {
return nil, err
}
_, _ = io.ReadAll(replay)
_ = replay.Close()
return nil, request.Context().Err()
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 1 {
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &originalCloses}, nil
}
return replaySource, nil
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := client.Transcribe(ctx, TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
})
done <- err
}()
select {
case <-replaySource.blocked:
cancel()
case <-time.After(time.Second):
cancel()
t.Fatal("replay source did not block")
}
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
}
case <-time.After(time.Second):
t.Fatal("Transcribe() did not return after replay cancellation")
}
if got := originalCloses.Load(); got != 1 {
t.Fatalf("original audio closes = %d, want 1", got)
}
select {
case <-replaySource.closed:
case <-time.After(time.Second):
t.Fatal("replay audio source was not closed")
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (roundTrip roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return roundTrip(request)
}
func newReplayTestHTTPClient(t *testing.T, transport http.RoundTripper) *HTTPClient {
t.Helper()
client, err := NewHTTPClient(HTTPClientConfig{
TranscribeURL: "http://whisperx.test/transcribe",
Language: "en",
Timeout: time.Second,
HTTPClient: &http.Client{Transport: transport},
})
if err != nil {
t.Fatalf("NewHTTPClient() error = %v", err)
}
return client
}
func whisperXJSONResponse(request *http.Request) *http.Response {
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"source":"transport-replay","segments":[]}`)),
Request: request,
}
}

View File

@@ -365,7 +365,7 @@ func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
}
}
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
func TestHTTPClientCancellationReleasesBlockedSource(t *testing.T) {
release := make(chan struct{})
source := newGatedReadCloser([]byte("audio-data"), release)
firstByteReceived := make(chan struct{})

View File

@@ -0,0 +1,263 @@
package whisperx
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"sync"
)
type multipartBodyFactory struct {
ctx context.Context
audioPath string
openAudio func(string) (io.ReadCloser, error)
prefix []byte
suffix []byte
contentType string
mu sync.Mutex
closed bool
bodies []*multipartBody
}
func newMultipartBodyFactory(
ctx context.Context,
audioPath string,
language string,
openAudio func(string) (io.ReadCloser, error),
) (*multipartBodyFactory, error) {
if ctx == nil {
ctx = context.Background()
}
if openAudio == nil {
return nil, fmt.Errorf("audio opener is required")
}
var framing bytes.Buffer
writer := multipart.NewWriter(&framing)
if _, err := writer.CreateFormFile("file", filepath.Base(audioPath)); err != nil {
return nil, fmt.Errorf("create multipart file field: %w", err)
}
prefixLength := framing.Len()
if err := writer.WriteField("language", language); err != nil {
return nil, fmt.Errorf("write language form field: %w", err)
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("close multipart framing: %w", err)
}
framingBytes := framing.Bytes()
return &multipartBodyFactory{
ctx: ctx,
audioPath: audioPath,
openAudio: openAudio,
prefix: append([]byte(nil), framingBytes[:prefixLength]...),
suffix: append([]byte(nil), framingBytes[prefixLength:]...),
contentType: writer.FormDataContentType(),
}, nil
}
func (factory *multipartBodyFactory) NewBody() (io.ReadCloser, error) {
factory.mu.Lock()
if factory.closed {
factory.mu.Unlock()
return nil, fmt.Errorf("multipart body factory is closed")
}
if err := factory.ctx.Err(); err != nil {
factory.mu.Unlock()
return nil, err
}
audio, err := factory.openAudio(factory.audioPath)
if err != nil {
factory.mu.Unlock()
return nil, fmt.Errorf("open audio file %q: %w", factory.audioPath, err)
}
if err := factory.ctx.Err(); err != nil {
factory.mu.Unlock()
_ = audio.Close()
return nil, err
}
source := &multipartAudioSource{audio: audio, closeDone: make(chan struct{})}
body := &multipartBody{
ctx: factory.ctx,
source: source,
reader: io.MultiReader(bytes.NewReader(factory.prefix), source, bytes.NewReader(factory.suffix)),
}
factory.bodies = append(factory.bodies, body)
factory.mu.Unlock()
stopCancel := context.AfterFunc(factory.ctx, func() {
_ = body.Close()
})
body.mu.Lock()
body.stopCancel = stopCancel
closed := body.closed
body.mu.Unlock()
if closed {
stopCancel()
}
if err := factory.ctx.Err(); err != nil {
_ = body.Close()
return nil, err
}
return body, nil
}
func (factory *multipartBodyFactory) Close() error {
factory.mu.Lock()
if factory.closed {
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
return multipartBodyCloseErrors(bodies)
}
factory.closed = true
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
for _, body := range bodies {
_ = body.Close()
}
return multipartBodyCloseErrors(bodies)
}
func (factory *multipartBodyFactory) SourceError(includeAbandoned bool) error {
factory.mu.Lock()
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
var result error
for _, body := range bodies {
if err := body.sourceError(includeAbandoned); err != nil {
result = errors.Join(result, err)
}
}
return result
}
func multipartBodyCloseErrors(bodies []*multipartBody) error {
var result error
for _, body := range bodies {
if err := body.sourceError(false); err != nil {
result = errors.Join(result, err)
}
}
return result
}
type multipartBody struct {
ctx context.Context
reader io.Reader
source *multipartAudioSource
stopCancel func() bool
mu sync.Mutex
closed bool
abandoned bool
}
func (body *multipartBody) Read(buffer []byte) (int, error) {
if err := body.ctx.Err(); err != nil {
return 0, err
}
body.mu.Lock()
closed := body.closed
body.mu.Unlock()
if closed {
return 0, io.ErrClosedPipe
}
return body.reader.Read(buffer)
}
func (body *multipartBody) Close() error {
body.mu.Lock()
if body.closed {
body.mu.Unlock()
return body.source.Close()
}
body.closed = true
body.abandoned = !body.source.reachedEOF()
stopCancel := body.stopCancel
body.mu.Unlock()
if stopCancel != nil {
stopCancel()
}
return body.source.Close()
}
func (body *multipartBody) sourceError(includeAbandoned bool) error {
body.mu.Lock()
abandoned := body.abandoned
body.mu.Unlock()
err := body.source.Err()
if err != nil {
return err
}
if abandoned && !includeAbandoned {
return nil
}
return nil
}
type multipartAudioSource struct {
audio io.ReadCloser
mu sync.Mutex
closing bool
eof bool
readErr error
closeErr error
closeDone chan struct{}
closeResult error
closed bool
}
func (source *multipartAudioSource) Read(buffer []byte) (int, error) {
n, err := source.audio.Read(buffer)
source.mu.Lock()
switch {
case errors.Is(err, io.EOF):
source.eof = true
case err != nil && !source.closing:
source.readErr = errors.Join(source.readErr, err)
}
source.mu.Unlock()
return n, err
}
func (source *multipartAudioSource) Close() error {
source.mu.Lock()
if source.closed {
done := source.closeDone
source.mu.Unlock()
<-done
source.mu.Lock()
err := source.closeResult
source.mu.Unlock()
return err
}
source.closed = true
source.closing = true
reachedEOF := source.eof
source.mu.Unlock()
err := source.audio.Close()
source.mu.Lock()
source.closeResult = err
if err != nil && reachedEOF {
source.closeErr = err
}
close(source.closeDone)
source.mu.Unlock()
return err
}
func (source *multipartAudioSource) reachedEOF() bool {
source.mu.Lock()
defer source.mu.Unlock()
return source.eof
}
func (source *multipartAudioSource) Err() error {
source.mu.Lock()
defer source.mu.Unlock()
return errors.Join(source.readErr, source.closeErr)
}

View File

@@ -0,0 +1,327 @@
package whisperx
import (
"bytes"
"context"
"errors"
"io"
"mime"
"mime/multipart"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestMultipartBodyReplayUsesIdenticalFraming(t *testing.T) {
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
defer factory.Close()
original, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() original error = %v", err)
}
replay, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() replay error = %v", err)
}
originalBytes, err := io.ReadAll(original)
if err != nil {
t.Fatalf("ReadAll(original) error = %v", err)
}
replayBytes, err := io.ReadAll(replay)
if err != nil {
t.Fatalf("ReadAll(replay) error = %v", err)
}
if !bytes.Equal(originalBytes, replayBytes) {
t.Fatal("original and replay multipart bytes differ")
}
parts := parseMultipartParts(t, factory.contentType, replayBytes)
if len(parts) != 2 {
t.Fatalf("part count = %d, want 2", len(parts))
}
if parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
t.Fatalf("file part = %#v, want file/audio.flac with exact audio", parts[0])
}
if parts[1].name != "language" || string(parts[1].data) != "en" {
t.Fatalf("language part = %#v, want language=en", parts[1])
}
}
func TestMultipartBodyReplayRestartsAfterPartialOriginal(t *testing.T) {
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
defer factory.Close()
original, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() original error = %v", err)
}
partial := make([]byte, len(factory.prefix)+3)
if _, err := io.ReadFull(original, partial); err != nil {
t.Fatalf("ReadFull(original) error = %v", err)
}
if err := original.Close(); err != nil {
t.Fatalf("Close(original) error = %v", err)
}
replay, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() replay error = %v", err)
}
replayBytes, err := io.ReadAll(replay)
if err != nil {
t.Fatalf("ReadAll(replay) error = %v", err)
}
parts := parseMultipartParts(t, factory.contentType, replayBytes)
if got := string(parts[0].data); got != "audio-data" {
t.Fatalf("replayed audio = %q, want complete payload", got)
}
if err := factory.SourceError(false); err != nil {
t.Fatalf("SourceError(false) = %v, want abandoned original ignored", err)
}
}
func TestMultipartBodySimultaneousBodiesUseIndependentSources(t *testing.T) {
var opens atomic.Int32
var closes atomic.Int32
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
opens.Add(1)
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
first, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() first error = %v", err)
}
second, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() second error = %v", err)
}
var firstBytes, secondBytes []byte
var firstErr, secondErr error
var group sync.WaitGroup
group.Add(2)
go func() {
defer group.Done()
firstBytes, firstErr = io.ReadAll(first)
}()
go func() {
defer group.Done()
secondBytes, secondErr = io.ReadAll(second)
}()
group.Wait()
if firstErr != nil || secondErr != nil {
t.Fatalf("concurrent reads returned (%v, %v)", firstErr, secondErr)
}
if !bytes.Equal(firstBytes, secondBytes) {
t.Fatal("independent body bytes differ")
}
if got := opens.Load(); got != 2 {
t.Fatalf("audio opens = %d, want 2", got)
}
if err := factory.Close(); err != nil {
t.Fatalf("Close() error = %v", err)
}
if err := factory.Close(); err != nil {
t.Fatalf("second Close() error = %v", err)
}
if got := closes.Load(); got != 2 {
t.Fatalf("audio closes = %d, want 2", got)
}
}
func TestMultipartBodyReportsSourceFailures(t *testing.T) {
t.Run("read", func(t *testing.T) {
sourceErr := errors.New("read source")
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
if _, err := io.ReadAll(body); !errors.Is(err, sourceErr) {
t.Fatalf("ReadAll() error = %v, want source failure", err)
}
_ = body.Close()
if err := factory.SourceError(false); !errors.Is(err, sourceErr) {
t.Fatalf("SourceError(false) = %v, want source failure", err)
}
})
t.Run("close after complete read", func(t *testing.T) {
closeErr := errors.New("close source")
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
return &errorCloseReadCloser{Reader: strings.NewReader("audio-data"), err: closeErr}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
if _, err := io.ReadAll(body); err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if err := body.Close(); !errors.Is(err, closeErr) {
t.Fatalf("Close() error = %v, want source close failure", err)
}
if err := factory.SourceError(false); !errors.Is(err, closeErr) {
t.Fatalf("SourceError(false) = %v, want source close failure", err)
}
})
}
func TestMultipartBodyCloseAndCancelReleaseBlockedSource(t *testing.T) {
for _, test := range []struct {
name string
release func(context.CancelFunc, io.ReadCloser)
}{
{name: "close", release: func(_ context.CancelFunc, body io.ReadCloser) { _ = body.Close() }},
{name: "cancel", release: func(cancel context.CancelFunc, _ io.ReadCloser) { cancel() }},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
source := newSignalingBlockedReadCloser()
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
return source, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
defer factory.Close()
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
readDone := make(chan error, 1)
go func() {
_, err := io.ReadAll(body)
readDone <- err
}()
select {
case <-source.blocked:
case <-time.After(time.Second):
t.Fatal("audio source did not block")
}
test.release(cancel, body)
select {
case <-source.closed:
case <-time.After(time.Second):
t.Fatal("audio source was not closed")
}
select {
case <-readDone:
case <-time.After(time.Second):
t.Fatal("blocked body read did not return")
}
if test.name == "cancel" {
if _, err := factory.NewBody(); !errors.Is(err, context.Canceled) {
t.Fatalf("NewBody() after cancellation error = %v, want context.Canceled", err)
}
}
})
}
}
func newTestMultipartBodyFactory(t *testing.T, ctx context.Context, audio string) *multipartBodyFactory {
t.Helper()
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader(audio)), nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
return factory
}
type multipartTestPart struct {
name string
filename string
data []byte
}
func parseMultipartParts(t *testing.T, contentType string, body []byte) []multipartTestPart {
t.Helper()
mediaType, params, err := mime.ParseMediaType(contentType)
if err != nil {
t.Fatalf("ParseMediaType() error = %v", err)
}
if mediaType != "multipart/form-data" {
t.Fatalf("media type = %q, want multipart/form-data", mediaType)
}
reader := multipart.NewReader(bytes.NewReader(body), params["boundary"])
var parts []multipartTestPart
for {
part, err := reader.NextPart()
if errors.Is(err, io.EOF) {
return parts
}
if err != nil {
t.Fatalf("NextPart() error = %v", err)
}
data, err := io.ReadAll(part)
if err != nil {
t.Fatalf("ReadAll(part %q) error = %v", part.FormName(), err)
}
parts = append(parts, multipartTestPart{name: part.FormName(), filename: part.FileName(), data: data})
}
}
type countedReadCloser struct {
io.Reader
closes *atomic.Int32
once sync.Once
}
func (reader *countedReadCloser) Close() error {
reader.once.Do(func() { reader.closes.Add(1) })
return nil
}
type errorCloseReadCloser struct {
io.Reader
err error
}
func (reader *errorCloseReadCloser) Close() error { return reader.err }
type signalingBlockedReadCloser struct {
blocked chan struct{}
closed chan struct{}
once sync.Once
read bool
}
func newSignalingBlockedReadCloser() *signalingBlockedReadCloser {
return &signalingBlockedReadCloser{blocked: make(chan struct{}), closed: make(chan struct{})}
}
func (reader *signalingBlockedReadCloser) Read(buffer []byte) (int, error) {
if !reader.read {
reader.read = true
return copy(buffer, "audio-data"), nil
}
reader.once.Do(func() { close(reader.blocked) })
<-reader.closed
return 0, errors.New("audio source closed")
}
func (reader *signalingBlockedReadCloser) Close() error {
select {
case <-reader.closed:
default:
close(reader.closed)
}
return nil
}

View File

@@ -272,13 +272,18 @@ func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
}
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
code := executeWithDependencies([]string{"analyze", "2026-05-03"}, &stdout, &stderr, deps, executor)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if got := stderr.String(); !strings.Contains(got, "analyze: no pipeline config path provided") ||
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
}
}
@@ -371,13 +376,18 @@ func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
}
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
code := executeWithDependencies([]string{"publish", "2026-05-03"}, &stdout, &stderr, deps, executor)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if got := stderr.String(); !strings.Contains(got, "publish: no pipeline config path provided") ||
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
}
}

View File

@@ -254,37 +254,37 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
var validated bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
t.Fatalf("validate production default: %v", err)
t.Fatalf("validate production base: %v", err)
}
if !strings.Contains(validated.String(), "profile=production") {
t.Fatalf("validate output = %q, want production profile", validated.String())
if !strings.Contains(validated.String(), "overlay=none") {
t.Fatalf("validate output = %q, want production base", validated.String())
}
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigShow, ConfigSources} {
if err := command(context.Background(), []string{
"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing",
"--config", pipelinePath, "--campaign-file", campaignPath, "--overlay", "testing",
}, io.Discard); err != nil {
t.Fatalf("testing inspection command %T: %v", command, err)
}
}
var diff bytes.Buffer
if err := ConfigDiff(context.Background(), []string{
"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
}, &diff); err != nil {
t.Fatalf("compare profiles: %v", err)
}
if !strings.Contains(diff.String(), "audita.model") || !strings.Contains(diff.String(), "character_items_arannis.profile_id") {
t.Fatalf("profile diff = %q, want model and expanded family changes", diff.String())
if !strings.Contains(diff.String(), "audita.proposal_profile") || !strings.Contains(diff.String(), "character_items_arannis.profile_id") {
t.Fatalf("profile diff = %q, want proposal profile and expanded family changes", diff.String())
}
common := []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath,
"--session", sessionPath, "--profile", "testing",
"--session", sessionPath, "--overlay", "testing",
}
var planned bytes.Buffer
if err := Plan(context.Background(), common, &planned); err != nil {
t.Fatalf("plan testing bundle: %v", err)
}
if !strings.Contains(planned.String(), "profile=testing") {
if !strings.Contains(planned.String(), "overlay=testing") {
t.Fatalf("plan output = %q, want testing provenance", planned.String())
}
@@ -305,7 +305,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
if !reflect.DeepEqual(capturedPlan.Names(), []string{"analyze"}) {
t.Fatalf("bounded plan = %#v, want analyze only", capturedPlan.Names())
}
if profile, ok := config.SelectedPipelineProfile(capturedCfg.Pipeline); !ok || profile.Name != "testing" {
if profile, ok := config.SelectedPipelineOverlay(capturedCfg.Pipeline); !ok || profile.Name != "testing" {
t.Fatalf("captured profile = %#v, selected=%t", profile, ok)
}
if digest := config.EffectivePipelineDigest(capturedCfg.Pipeline); digest == "" || !strings.Contains(runOut.String(), "digest="+digest) {
@@ -365,7 +365,7 @@ func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *tes
if !reflect.DeepEqual(productionRuns, make([]int, len(names))) {
t.Fatalf("production runs = %v, want complete reuse", productionRuns)
}
assertAssembledProvenance(t, store, productionSummary, "production", config.EffectivePipelineDigest(production.Pipeline))
assertAssembledProvenance(t, store, productionSummary, "", config.EffectivePipelineDigest(production.Pipeline))
testingRuns := make([]int, len(names))
testingStages := assembledSemanticStages(names, providers, testingRuns)
@@ -391,7 +391,7 @@ func loadAssembledSplitBundleConfig(t *testing.T, profile string) *config.Config
pipelinePath, campaignPath, sessionPath := assembledSplitBundlePaths()
options := config.SessionLoadOptions{}
if profile != "" {
options.Profile = &profile
options.Overlay = &profile
}
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, options)
if err != nil {
@@ -415,7 +415,9 @@ func assertAssembledProvenance(t *testing.T, store *manifest.LocalStore, summary
t.Fatal(err)
}
value := session.EffectiveConfig
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
profileMatches := value != nil && ((profile == "" && value.SelectedOverlay == nil) ||
(profile != "" && value.SelectedOverlay != nil && value.SelectedOverlay.Name == profile))
if !profileMatches || value.EffectiveConfigDigest != digest {
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
}
}

View File

@@ -57,7 +57,7 @@ func TestBoundedRunParsingRejectsDuplicateSingletons(t *testing.T) {
{name: "through mixed", args: []string{"session", "--through", "analyze", "--through=publish"}, want: "--through may be specified only once"},
{name: "force separate", args: []string{"session", "--force", "--force"}, want: "--force may be specified only once"},
{name: "force equals", args: []string{"session", "--force=true", "--force=false"}, want: "--force may be specified only once"},
{name: "profile", args: []string{"session", "--profile", "production", "--profile=testing"}, want: "--profile may be specified only once"},
{name: "overlay", args: []string{"session", "--overlay", "production", "--overlay=testing"}, want: "--overlay may be specified only once"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
@@ -69,29 +69,29 @@ func TestBoundedRunParsingRejectsDuplicateSingletons(t *testing.T) {
}
}
func TestBoundedRunParsingRetainsExplicitProfilePresence(t *testing.T) {
withoutProfile, err := parseBoundedRunRequest("run", []string{"session"}, io.Discard)
func TestBoundedRunParsingRetainsExplicitOverlayPresence(t *testing.T) {
withoutOverlay, err := parseBoundedRunRequest("run", []string{"session"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if got := withoutProfile.Config.sessionOptions().Profile; got != nil {
t.Fatalf("omitted profile = %#v, want nil", got)
if got := withoutOverlay.Config.sessionOptions().Overlay; got != nil {
t.Fatalf("omitted overlay = %#v, want nil", got)
}
withProfile, err := parseBoundedRunRequest("run", []string{"session", "--profile", "testing"}, io.Discard)
withOverlay, err := parseBoundedRunRequest("run", []string{"session", "--overlay", "testing"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if got := withProfile.Config.sessionOptions().Profile; got == nil || *got != "testing" {
t.Fatalf("explicit profile = %#v, want testing", got)
if got := withOverlay.Config.sessionOptions().Overlay; got == nil || *got != "testing" {
t.Fatalf("explicit overlay = %#v, want testing", got)
}
explicitEmpty, err := parseBoundedRunRequest("run", []string{"session", "--profile", ""}, io.Discard)
explicitEmpty, err := parseBoundedRunRequest("run", []string{"session", "--overlay", ""}, io.Discard)
if err != nil {
t.Fatal(err)
}
if got := explicitEmpty.Config.sessionOptions().Profile; got == nil || *got != "" {
t.Fatalf("explicit empty profile = %#v, want non-nil empty", got)
if got := explicitEmpty.Config.sessionOptions().Overlay; got == nil || *got != "" {
t.Fatalf("explicit empty overlay = %#v, want non-nil empty", got)
}
}

View File

@@ -45,7 +45,7 @@ func cleanSessionWithDependencies(ctx context.Context, flags commonConfigFlags,
if strings.TrimSpace(flags.sessionID) == "" {
return fmt.Errorf("clean: session_id is required unless --all is set")
}
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
if err != nil {
return fmt.Errorf("clean: %w", err)
}
@@ -93,7 +93,7 @@ func cleanAllLocalWithDependencies(flags commonConfigFlags, dryRun, clearCache b
strings.TrimSpace(flags.previousSessionID) != "" {
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
}
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Overlay: flags.overlay.pointer()}, flags.secretsDir.pointer(), deps)
if err != nil {
return fmt.Errorf("clean: %w", err)
}

View File

@@ -2,16 +2,20 @@ package app
import (
"context"
"os"
"io"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
type commandDependencies struct {
diagnostics io.Writer
resolvePipelineConfigPath func(string) (string, error)
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
readSecretsDirectory func(string) ([]os.DirEntry, error)
loadSecretsDir func(string) (*configkit.SecretsReport, error)
secretsBootstrap *secretsBootstrap
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
@@ -19,14 +23,23 @@ type commandDependencies struct {
}
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
if deps.diagnostics == nil {
deps.diagnostics = io.Discard
}
if deps.resolvePipelineConfigPath == nil {
deps.resolvePipelineConfigPath = resolvePipelineConfigPath
}
if deps.loadPipelineConfig == nil {
deps.loadPipelineConfig = config.LoadPipelineWithOptions
}
if deps.downloadObjectToTemp == nil {
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
}
if deps.readSecretsDirectory == nil {
deps.readSecretsDirectory = os.ReadDir
if deps.loadSecretsDir == nil {
deps.loadSecretsDir = configkit.LoadSecretsDir
}
if deps.secretsBootstrap == nil {
deps.secretsBootstrap = &secretsBootstrap{}
}
if deps.newObjectStore == nil {
deps.newObjectStore = storage.NewObjectStoreFromConfig

View File

@@ -19,6 +19,9 @@ func executeWithPipelineExecutor(args []string, stdout, stderr io.Writer, execut
}
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
if deps.diagnostics == nil {
deps.diagnostics = stderr
}
deps = normalizeCommandDependencies(deps)
if len(args) == 0 {
printUsage(stderr)

View File

@@ -59,9 +59,10 @@ func TestExecuteValidCommands(t *testing.T) {
func TestExecuteMissingRequiredFlags(t *testing.T) {
cases := []struct {
name string
args []string
want string
name string
args []string
want string
controlledDiscovery bool
}{
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
@@ -69,7 +70,7 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided", controlledDiscovery: true},
}
for _, tc := range cases {
@@ -77,7 +78,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute(tc.args, &stdout, &stderr)
deps := commandDependencies{}
executor := pipelineExecutor(executePlan)
missingPath := ""
if tc.controlledDiscovery {
deps, missingPath, executor = guardedMissingPipelineDiscovery(t)
}
code := executeWithDependencies(tc.args, &stdout, &stderr, deps, executor)
if code == 0 {
t.Fatalf("exit code = 0, want non-zero")
}
@@ -87,6 +94,11 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
if !strings.Contains(stderr.String(), tc.want) {
t.Fatalf("stderr = %q, want to contain %q", stderr.String(), tc.want)
}
if tc.controlledDiscovery {
if got := stderr.String(); !strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
t.Fatalf("stderr = %q, want controlled searched path %q", got, missingPath)
}
}
})
}
}
@@ -192,7 +204,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
configDir := t.TempDir()
sessionID := "2026-05-03"
secretsDir := filepath.Join(configDir, "secrets")
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
}
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
@@ -211,15 +223,13 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
root: ` + workspaceRoot + `
storage:
backend: local
secrets:
env_dir: ./secrets
whisperx:
transcribe_url: https://example.com/transcribe
seriatim:
binary: ` + seriatimBinary + `
audita:
binary: ` + auditaBinary + `
llm_api_key_env: OPENROUTER_API_KEY
credentials: [OPENROUTER_API_KEY]
notification:
mode: noop
`
@@ -264,9 +274,53 @@ inputs:
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
t.Fatalf("stdout = %q, want polish execution", stdout.String())
}
recorded, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(workRoot, "manifest.json"))
if err != nil {
t.Fatalf("load polished manifest: %v", err)
}
polish := recorded.Stages["polish"]
if polish == nil || len(polish.Outputs) != 1 || polish.Outputs[0].Kind != "transcript_polished" {
t.Fatalf("polish outputs = %#v, want only canonical transcript", polish)
}
if len(polish.Logs) != 5 || len(polish.GeneratedConfigs) != 1 {
t.Fatalf("polish diagnostics = %#v, want logs, reports, and invocation", polish)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
found := false
for _, path := range polish.Logs {
if filepath.Base(path) == name {
found = true
}
}
if !found {
t.Fatalf("report %q absent from manifest diagnostics: %#v", name, polish.Logs)
}
}
foundReference := false
if err := filepath.WalkDir(workRoot, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil || entry.IsDir() {
return walkErr
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
if bytes.Contains(data, []byte("from-secret-file")) {
t.Fatalf("workspace output %q exposed credential value", path)
}
if bytes.Contains(data, []byte("OPENROUTER_API_KEY")) {
foundReference = true
}
return nil
}); err != nil {
t.Fatalf("inspect polish outputs: %v", err)
}
if !foundReference {
t.Fatal("polish outputs omitted safe credential reference")
}
}
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
func TestExecuteRunFailsWhenExplicitSecretsDirMissing(t *testing.T) {
workspaceRoot := t.TempDir()
configDir := t.TempDir()
pipelinePath := filepath.Join(configDir, "pipeline.yml")
@@ -277,8 +331,6 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
root: ` + workspaceRoot + `
storage:
backend: local
secrets:
env_dir: ./missing-secrets
whisperx:
transcribe_url: https://example.com/transcribe
seriatim:
@@ -305,11 +357,11 @@ inputs:
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--secrets-dir", filepath.Join(configDir, "missing-secrets")}, &stdout, &stderr)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
if !strings.Contains(stderr.String(), "bootstrap secrets directory") {
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
}
}
@@ -748,10 +800,18 @@ func TestAuditaAppHelper(t *testing.T) {
os.Exit(2)
}
processArgs := args[start:]
if len(processArgs) < 2 || processArgs[0] != "process" {
_, _ = os.Stderr.WriteString("expected Audita v2 process invocation\n")
os.Exit(2)
}
for _, arg := range processArgs {
if arg == "--report-json" || arg == "--work-dir" || arg == "--debug" || arg == "--reporting=false" {
_, _ = os.Stderr.WriteString("retired or unwanted Audita flag\n")
os.Exit(2)
}
}
outputPath := appSeriatimFlagValue(processArgs, "--output")
reportPath := appSeriatimFlagValue(processArgs, "--report-json")
workDir := appSeriatimFlagValue(processArgs, "--work-dir")
if strings.TrimSpace(outputPath) == "" {
_, _ = os.Stderr.WriteString("missing --output\n")
os.Exit(2)
@@ -764,19 +824,9 @@ func TestAuditaAppHelper(t *testing.T) {
_, _ = os.Stderr.WriteString(fmt.Sprintf("write output: %v\n", err))
os.Exit(2)
}
if strings.TrimSpace(reportPath) != "" {
if err := os.MkdirAll(filepath.Dir(reportPath), 0o755); err != nil {
_, _ = os.Stderr.WriteString(fmt.Sprintf("mkdir report dir: %v\n", err))
os.Exit(2)
}
if err := os.WriteFile(reportPath, []byte(`{"report":true}`), 0o644); err != nil {
_, _ = os.Stderr.WriteString(fmt.Sprintf("write report: %v\n", err))
os.Exit(2)
}
}
if strings.TrimSpace(workDir) != "" {
if err := os.MkdirAll(workDir, 0o755); err != nil {
_, _ = os.Stderr.WriteString(fmt.Sprintf("mkdir work dir: %v\n", err))
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if err := os.WriteFile(filepath.Join(filepath.Dir(outputPath), name), []byte("status: completed\n"), 0o644); err != nil {
_, _ = os.Stderr.WriteString(fmt.Sprintf("write %s: %v\n", name, err))
os.Exit(2)
}
}

View File

@@ -0,0 +1,211 @@
package app
import (
"bytes"
"context"
"errors"
"path/filepath"
"reflect"
"strings"
"testing"
)
const artifactCatalogHeader = "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source\n"
func TestConfigArtifactsReportsOrdinaryCatalogWithoutCampaign(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, `workspace:
root: `+filepath.Join(dir, "workspace")+`
whisperx:
transcribe_url: https://transcription.example.com/transcribe
scriptorium:
binary: scriptorium
artifacts:
zeta:
enabled: false
output_path: artifacts/zeta.md
alpha:
enabled: false
output_path: artifacts/alpha.md
inputs:
upstream: {source: narratio.artifact.zeta, required: true}
`)
var out bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
t.Fatalf("ConfigArtifacts() error = %v", err)
}
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
if len(lines) != 3 || !strings.HasPrefix(lines[1], "alpha\t") || !strings.HasPrefix(lines[2], "zeta\t") {
t.Fatalf("catalog order/output = %q", out.String())
}
alpha := strings.Split(lines[1], "\t")
if len(alpha) != 10 {
t.Fatalf("alpha fields = %#v", alpha)
}
if alpha[1] != "narratio.artifact.alpha" || alpha[2] != pipelinePath || alpha[3] != "false" || alpha[4] != "artifacts/alpha.md" || alpha[5] != "zeta" {
t.Fatalf("alpha record = %#v", alpha)
}
}
func TestConfigArtifactsEmptyCatalogWritesOnlyHeader(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
var out bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
t.Fatal(err)
}
if out.String() != artifactCatalogHeader {
t.Fatalf("empty catalog = %q, want header only", out.String())
}
}
func TestConfigArtifactsReportsGeneratedFamilyProvenance(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
var out bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
t.Fatal(err)
}
records := catalogRecordsByKey(t, out.String())
meta := records["character_meta_arannis"]
note := records["character_note_arannis"]
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
if len(meta) != 10 || meta[6] != "character_meta" || meta[7] != "arannis" || meta[8] != pipelinePath || meta[9] != partyPath {
t.Fatalf("generated meta record = %#v", meta)
}
if len(note) != 10 || note[5] != "character_meta_arannis" || note[6] != "character_note" {
t.Fatalf("generated note record = %#v", note)
}
}
func TestConfigArtifactsReportsOverlayFamilyContributors(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
overlayPath := filepath.Join(filepath.Dir(pipelinePath), "overlays", "testing.yml")
mustWriteTestFile(t, overlayPath, `scriptorium:
artifact_families:
character_note:
prompt_id: dnd.character_note.testing
`)
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--overlay", "testing"}
var catalog bytes.Buffer
if err := ConfigArtifacts(context.Background(), args, &catalog); err != nil {
t.Fatal(err)
}
records := catalogRecordsByKey(t, catalog.String())
if got := strings.Split(records["character_note_arannis"][2], ","); !reflect.DeepEqual(got, []string{overlayPath, pipelinePath}) {
t.Fatalf("generated family definitions = %#v", got)
}
var sources bytes.Buffer
if err := ConfigSources(context.Background(), args, &sources); err != nil {
t.Fatal(err)
}
want := "scriptorium.artifacts.character_note_arannis.prompt_id\tfamily\t" + overlayPath
if !strings.Contains(sources.String(), want) {
t.Fatalf("sources output missing overlay family contributor %q:\n%s", want, sources.String())
}
}
func TestConfigArtifactsReportsBaseAndOverlayContributors(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
overlayPath := filepath.Join(dir, "overlays", "testing.yml")
mustWriteTestFile(t, pipelinePath, `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
scriptorium:
binary: scriptorium
artifacts:
notes:
enabled: false
output_path: artifacts/notes.md
`)
mustWriteTestFile(t, overlayPath, `scriptorium:
artifacts:
notes:
enabled: true
prompt_id: notes
overlay_only:
enabled: false
output_path: artifacts/overlay-only.md
`)
var out bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--overlay", "testing"}, &out); err != nil {
t.Fatal(err)
}
records := catalogRecordsByKey(t, out.String())
if got := strings.Split(records["notes"][2], ","); !reflect.DeepEqual(got, []string{overlayPath, pipelinePath}) {
t.Fatalf("notes definitions = %#v", got)
}
if got := records["overlay_only"][2]; got != overlayPath {
t.Fatalf("overlay-only definition = %q, want %q", got, overlayPath)
}
}
func TestConfigArtifactsIdentityDoesNotDependOnDefinitionFilename(t *testing.T) {
inlineDir := t.TempDir()
inlinePath := filepath.Join(inlineDir, "pipeline.yml")
mustWriteTestFile(t, inlinePath, artifactIdentityPipeline("", "recap"))
importDir := t.TempDir()
importPath := filepath.Join(importDir, "pipeline.yml")
mustWriteTestFile(t, importPath, artifactIdentityPipeline("artifacts.yml", ""))
mustWriteTestFile(t, filepath.Join(importDir, "artifacts.yml"), "scriptorium:\n artifacts:\n recap:\n enabled: false\n output_path: artifacts/recap.md\n")
for _, path := range []string{inlinePath, importPath} {
var out bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", path}, &out); err != nil {
t.Fatal(err)
}
if got := catalogRecordsByKey(t, out.String())["recap"][1]; got != "narratio.artifact.recap" {
t.Fatalf("source identity for %s = %q", path, got)
}
}
}
func TestConfigArtifactsPropagatesWriterErrorsAndEscapesFields(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
want := errors.New("write rejected")
err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, rejectingCatalogWriter{err: want})
if !errors.Is(err, want) {
t.Fatalf("ConfigArtifacts() error = %v, want writer error", err)
}
if got := escapeCatalogField("a\\b\tc\nd\r"); got != `a\\b\tc\nd\r` {
t.Fatalf("escaped field = %q", got)
}
}
func catalogRecordsByKey(t *testing.T, output string) map[string][]string {
t.Helper()
lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n")
if len(lines) == 0 || lines[0]+"\n" != artifactCatalogHeader {
t.Fatalf("catalog header = %q", output)
}
records := make(map[string][]string, len(lines)-1)
for _, line := range lines[1:] {
fields := strings.Split(line, "\t")
records[fields[0]] = fields
}
return records
}
func artifactIdentityPipeline(importPath, inlineKey string) string {
document := "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\nscriptorium:\n binary: scriptorium\n"
if importPath != "" {
document = "configkit:\n imports:\n - path: " + importPath + "\n" + document
}
if inlineKey != "" {
document += " artifacts:\n " + inlineKey + ":\n enabled: false\n output_path: artifacts/recap.md\n"
}
return document
}
type rejectingCatalogWriter struct{ err error }
func (w rejectingCatalogWriter) Write([]byte) (int, error) { return 0, w.err }

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"sort"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/config"
@@ -16,7 +17,8 @@ type inspectionFlags struct {
pipelinePath string
campaignPath string
campaignFilePath string
profile singletonStringFlag
overlay singletonStringFlag
secretsDir singletonStringFlag
}
type inspectionConfig struct {
@@ -35,7 +37,7 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
deps = normalizeCommandDependencies(deps)
if len(args) == 0 {
return fmt.Errorf("config: expected subcommand: validate|show|sources|diff")
return fmt.Errorf("config: expected subcommand: validate|show|sources|artifacts|diff")
}
if args[0] == "--help" || args[0] == "-h" {
writeConfigCommandUsage(out)
@@ -48,8 +50,10 @@ func configWithDependencies(ctx context.Context, args []string, out io.Writer, d
return configShowWithDependencies(ctx, args[1:], out, deps)
case "sources":
return configSourcesWithDependencies(ctx, args[1:], out, deps)
case "artifacts":
return configArtifactsWithDependencies(ctx, args[1:], out, deps)
case "diff":
return ConfigDiff(ctx, args[1:], out)
return configDiffWithDependencies(ctx, args[1:], out, deps)
default:
return fmt.Errorf("config: unknown subcommand %q", args[0])
}
@@ -73,8 +77,8 @@ func configValidateWithDependencies(_ context.Context, args []string, out io.Wri
if err != nil {
return fmt.Errorf("config validate: %w", err)
}
_, err = fmt.Fprintf(out, "Configuration valid: root=%s; profile=%s; digest=%s\n",
inspectionRootPath(resolved), inspectionProfile(resolved.Pipeline), config.EffectivePipelineDigest(resolved.Pipeline))
_, err = fmt.Fprintf(out, "Configuration valid: root=%s; overlay=%s; digest=%s\n",
inspectionRootPath(resolved), inspectionOverlay(resolved.Pipeline), config.EffectivePipelineDigest(resolved.Pipeline))
return err
}
@@ -139,6 +143,70 @@ func configSourcesWithDependencies(_ context.Context, args []string, out io.Writ
return nil
}
// ConfigArtifacts validates and reports the effective concrete artifact catalog
// without loading session state or runtime collaborators.
func ConfigArtifacts(ctx context.Context, args []string, out io.Writer) error {
return configArtifactsWithDependencies(ctx, args, out, commandDependencies{})
}
func configArtifactsWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
flags, err := parseInspectionFlags("config artifacts", args, out)
if err != nil {
if errors.Is(err, flag.ErrHelp) {
return nil
}
return err
}
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
if err != nil {
return fmt.Errorf("config artifacts: %w", err)
}
records, err := config.EffectiveArtifactCatalog(resolved.Pipeline, resolved.Party)
if err != nil {
return fmt.Errorf("config artifacts: %w", err)
}
if _, err := fmt.Fprintln(out, "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source"); err != nil {
return err
}
for _, record := range records {
fields := []string{
record.Key,
record.SourceID,
strings.Join(normalizedCatalogPaths(record.DefinitionSources), ","),
strconv.FormatBool(record.Enabled),
record.OutputPath,
strings.Join(record.Dependencies, ","),
record.FamilyKey,
record.CharacterID,
config.NormalizedConfigurationPath(record.FamilySource),
config.NormalizedConfigurationPath(record.PartySource),
}
for index := range fields {
fields[index] = escapeCatalogField(fields[index])
}
if _, err := fmt.Fprintln(out, strings.Join(fields, "\t")); err != nil {
return err
}
}
return nil
}
func normalizedCatalogPaths(paths []string) []string {
result := make([]string, len(paths))
for index, path := range paths {
result[index] = config.NormalizedConfigurationPath(path)
}
sort.Strings(result)
return result
}
func escapeCatalogField(value string) string {
value = strings.ReplaceAll(value, "\\", "\\\\")
value = strings.ReplaceAll(value, "\t", "\\t")
value = strings.ReplaceAll(value, "\r", "\\r")
return strings.ReplaceAll(value, "\n", "\\n")
}
func parseInspectionFlags(command string, args []string, out io.Writer) (inspectionFlags, error) {
fs := flag.NewFlagSet(command, flag.ContinueOnError)
fs.SetOutput(out)
@@ -146,10 +214,12 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&flags.campaignPath, "campaign", "", "campaign ID")
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
flags.profile.name = "profile"
fs.Var(&flags.profile, "profile", "named pipeline profile")
flags.overlay.name = "overlay"
fs.Var(&flags.overlay, "overlay", "named pipeline overlay")
flags.secretsDir.name = "secrets-dir"
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
fs.Usage = func() {
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]\n\n", command)
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--overlay <name>] [--secrets-dir <path>]\n\n", command)
fmt.Fprintln(out, "Resolves configuration without a session, workspace, manifest, adapters, or external services.")
fmt.Fprintln(out)
fmt.Fprintln(out, "Flags:")
@@ -168,7 +238,7 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
}
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Overlay: flags.overlay.pointer()}, flags.secretsDir.pointer(), deps)
if err != nil {
return nil, err
}
@@ -245,17 +315,17 @@ func inspectionRootPath(resolved *inspectionConfig) string {
return resolved.PipelinePath
}
func inspectionProfile(pipeline *config.PipelineConfig) string {
if profile, ok := config.SelectedPipelineProfile(pipeline); ok {
return profile.Name
func inspectionOverlay(pipeline *config.PipelineConfig) string {
if overlay, ok := config.SelectedPipelineOverlay(pipeline); ok {
return overlay.Name
}
return "none"
}
func writeConfigCommandUsage(out io.Writer) {
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|diff>")
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|artifacts|diff>")
fmt.Fprintln(out)
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, or config diff to compare two profiles.")
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, config artifacts to list concrete artifacts, or config diff to compare two overlays.")
}
func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) error {
@@ -273,11 +343,11 @@ func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) erro
return err
}
}
if profile, ok := config.SelectedPipelineProfile(resolved.Pipeline); ok {
if _, err := fmt.Fprintf(out, "profile: name=%s selection=%s overlay=%s\n", profile.Name, profile.Source, config.NormalizedConfigurationPath(profile.OverlayPath)); err != nil {
if overlay, ok := config.SelectedPipelineOverlay(resolved.Pipeline); ok {
if _, err := fmt.Fprintf(out, "overlay: name=%s selection=%s path=%s\n", overlay.Name, overlay.Source, config.NormalizedConfigurationPath(overlay.OverlayPath)); err != nil {
return err
}
} else if _, err := fmt.Fprintln(out, "profile: none"); err != nil {
} else if _, err := fmt.Fprintln(out, "overlay: none"); err != nil {
return err
}
if resolved.Campaign == nil {

View File

@@ -11,6 +11,7 @@ import (
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
@@ -21,17 +22,19 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
want string
code int
}{
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|diff>"},
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|artifacts|diff>"},
{name: "validate help", args: []string{"config", "validate", "--help"}, want: "Usage: narratio config validate"},
{name: "show help", args: []string{"config", "show", "--help"}, want: "Usage: narratio config show"},
{name: "sources help", args: []string{"config", "sources", "--help"}, want: "Usage: narratio config sources"},
{name: "artifacts help", args: []string{"config", "artifacts", "--help"}, want: "Usage: narratio config artifacts"},
{name: "diff help", args: []string{"config", "diff", "--help"}, want: "Usage: narratio config diff"},
{name: "unknown", args: []string{"config", "unknown"}, want: `config: unknown subcommand "unknown"`, code: 1},
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|diff", code: 1},
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|artifacts|diff", code: 1},
{name: "session flag", args: []string{"config", "validate", "--session", "session.yml"}, want: "flag provided but not defined", code: 1},
{name: "stage flag", args: []string{"config", "show", "--from", "prepare"}, want: "flag provided but not defined", code: 1},
{name: "force flag", args: []string{"config", "show", "--force"}, want: "flag provided but not defined", code: 1},
{name: "artifact flag", args: []string{"config", "show", "--artifacts", "recap"}, want: "flag provided but not defined", code: 1},
{name: "retired selector", args: []string{"config", "validate", "--profile", "testing"}, want: "flag provided but not defined", code: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
@@ -52,7 +55,7 @@ func TestExecuteConfigUsesInvocationPipelineLoader(t *testing.T) {
pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
for _, subcommand := range []string{"validate", "show", "sources"} {
for _, subcommand := range []string{"validate", "show", "sources", "artifacts"} {
t.Run(subcommand, func(t *testing.T) {
calls := 0
deps := commandDependencies{
@@ -91,8 +94,6 @@ func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
defer server.Close()
mustWriteTestFile(t, pipelinePath, `workspace:
root: `+workspaceRoot+`
secrets:
env_dir: `+secretsDir+`
whisperx:
transcribe_url: `+server.URL+`
`)
@@ -102,7 +103,7 @@ whisperx:
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &validateOut); err != nil {
t.Fatalf("ConfigValidate() error = %v", err)
}
if !strings.Contains(validateOut.String(), "Configuration valid: root=") || !strings.Contains(validateOut.String(), "; profile=none; digest=") {
if !strings.Contains(validateOut.String(), "Configuration valid: root=") || !strings.Contains(validateOut.String(), "; overlay=none; digest=") {
t.Fatalf("validate output = %q", validateOut.String())
}
@@ -127,7 +128,7 @@ whisperx:
if strings.Contains(sourcesOut.String(), secret) {
t.Fatalf("sources output leaked a raw secret: %q", sourcesOut.String())
}
writeInspectionProfiles(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
writeInspectionOverlays(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
var diffOut bytes.Buffer
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diffOut); err != nil {
t.Fatalf("ConfigDiff() error = %v", err)
@@ -143,10 +144,10 @@ whisperx:
}
}
func TestConfigDiffReportsSemanticProfileChanges(t *testing.T) {
func TestConfigDiffReportsSemanticOverlayChanges(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
writeInspectionProfiles(t, pipelinePath, `workspace:
writeInspectionOverlays(t, pipelinePath, `workspace:
cleanup_after_publish: true
publish:
upload_run: true
@@ -203,13 +204,63 @@ scriptorium:
t.Fatalf("reversed ConfigDiff() error = %v", err)
}
if !strings.Contains(reversed.String(), "changed\twhisperx.language\t\"fr\"\t\"en\"\n") {
t.Fatalf("reversed diff did not independently resolve profiles:\n%s", reversed.String())
t.Fatalf("reversed diff did not independently resolve overlays:\n%s", reversed.String())
}
}
func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, pipelinePath, `workspace:
root: /tmp/narratio-work
whisperx:
transcribe_url: https://transcription.example.test/transcribe
audita:
modules: null
scriptorium:
artifacts: null
`)
mustWriteTestFile(t, filepath.Join(dir, "overlays", "empty.yml"), `audita:
modules: []
scriptorium:
artifacts: {}
`)
var base, empty bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil {
t.Fatal(err)
}
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--overlay", "empty"}, &empty); err != nil {
t.Fatal(err)
}
for _, want := range []string{"modules: null\n", "artifacts: null\n"} {
if !strings.Contains(base.String(), want) {
t.Fatalf("base show output missing %q:\n%s", want, base.String())
}
}
for _, want := range []string{"modules: []\n", "artifacts: {}\n"} {
if !strings.Contains(empty.String(), want) {
t.Fatalf("empty-overlay show output missing %q:\n%s", want, empty.String())
}
}
var diff bytes.Buffer
if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil {
t.Fatal(err)
}
for _, want := range []string{
"changed\taudita.modules\tnull\t[]\n",
"changed\tscriptorium.artifacts\tnull\t{}\n",
} {
if !strings.Contains(diff.String(), want) {
t.Fatalf("diff output missing %q:\n%s", want, diff.String())
}
}
}
func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
writeInspectionProfiles(t, pipelinePath, `scriptorium:
writeInspectionOverlays(t, pipelinePath, `scriptorium:
artifact_families:
character_note:
enabled: false
@@ -251,11 +302,11 @@ func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
writeInspectionProfiles(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: en\n")
writeInspectionOverlays(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: en\n")
var equal bytes.Buffer
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &equal); err != nil {
t.Fatalf("ConfigDiff() equal profiles error = %v", err)
t.Fatalf("ConfigDiff() equal overlays error = %v", err)
}
if got := equal.String(); got != "no differences\n" {
t.Fatalf("equal diff output = %q", got)
@@ -266,7 +317,7 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
{"production", "--config", pipelinePath},
{"production", "production", "--config", pipelinePath},
{"unknown", "testing", "--config", pipelinePath},
{"production", "testing", "--config", pipelinePath, "--profile", "production"},
{"production", "testing", "--config", pipelinePath, "--overlay", "production"},
{"production", "testing", "--config", pipelinePath, "--config", pipelinePath},
{"production", "testing", "--config", pipelinePath, "--campaign", "sample-campaign", "--campaign-file", campaignPath},
} {
@@ -276,6 +327,137 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
}
}
func TestConfigDiffBootstrapsSecretsOnceForBothOverlays(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
writeInspectionOverlays(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
secretsDir := t.TempDir()
const secretName = "NARRATIO_DIFF_BOOTSTRAP_REPORT"
restoreEnvAfterTest(t, secretName)
if err := os.Unsetenv(secretName); err != nil {
t.Fatal(err)
}
mustWriteSecretFile(t, filepath.Join(secretsDir, secretName), "diff-secret-must-not-appear\n")
bootstrapCalls := 0
var diagnostics bytes.Buffer
deps := commandDependencies{loadSecretsDir: func(got string) (*configkit.SecretsReport, error) {
bootstrapCalls++
if got != secretsDir {
t.Fatalf("secrets directory = %q, want %q", got, secretsDir)
}
return configkit.LoadSecretsDir(got)
}, diagnostics: &diagnostics}
if err := configDiffWithDependencies(context.Background(), []string{
"production", "testing", "--config", pipelinePath, "--secrets-dir", secretsDir,
}, io.Discard, deps); err != nil {
t.Fatalf("config diff: %v", err)
}
if bootstrapCalls != 1 {
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
}
if got := diagnostics.String(); strings.Count(got, `name="`+secretName+`"`) != 1 {
t.Fatalf("bootstrap report count = %d, want 1; diagnostics = %q", strings.Count(got, `name="`+secretName+`"`), got)
}
if strings.Contains(diagnostics.String(), "diff-secret-must-not-appear") {
t.Fatalf("bootstrap diagnostics exposed secret value: %q", diagnostics.String())
}
}
func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T) {
const firstCanary = "inspection-credential-canary-one"
const secondCanary = "inspection-credential-canary-two"
for name, value := range map[string]string{
"NARRATIO_INSPECT_ACCESS": firstCanary,
"NARRATIO_INSPECT_SECRET": firstCanary,
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
"NARRATIO_INSPECT_NOTARIUS_ONE": firstCanary,
"NARRATIO_INSPECT_NOTARIUS_TWO": secondCanary,
} {
t.Setenv(name, value)
}
workspaceRoot := t.TempDir()
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
pipelineData, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
pipelineYAML := strings.Replace(string(pipelineData), " bucket: test-bucket\n", ` bucket: test-bucket
access_key_id: NARRATIO_INSPECT_ACCESS
secret_access_key: NARRATIO_INSPECT_SECRET
`, 1)
pipelineYAML += `notarius:
enabled: false
credentials: [NARRATIO_INSPECT_NOTARIUS_ONE]
`
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatal(err)
}
writeInspectionOverlays(t, pipelinePath,
"audita:\n proposal_profile: production\n credentials: [NARRATIO_INSPECT_LLM_ONE]\n",
"audita:\n proposal_profile: testing\n credentials: [NARRATIO_INSPECT_LLM_TWO]\nnotarius:\n credentials: [NARRATIO_INSPECT_NOTARIUS_TWO]\n",
)
mustWriteTestFile(t, filepath.Join(filepath.Dir(pipelinePath), "overlays", "without-notarius-key.yml"), "notarius:\n credentials: []\n")
var validate bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--overlay", "production"}, &validate); err != nil {
t.Fatalf("config validate: %v", err)
}
var show bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--overlay", "production"}, &show); err != nil {
t.Fatalf("config show: %v", err)
}
var sources bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--overlay", "production"}, &sources); err != nil {
t.Fatalf("config sources: %v", err)
}
var catalog bytes.Buffer
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--overlay", "production"}, &catalog); err != nil {
t.Fatalf("config artifacts: %v", err)
}
var diff bytes.Buffer
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil {
t.Fatalf("config diff: %v", err)
}
var removed bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--overlay", "without-notarius-key"}, &removed); err != nil {
t.Fatalf("config show with removed Notarius credential: %v", err)
}
for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} {
if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) {
t.Fatalf("%s output exposed credential value: %q", label, output)
}
}
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE", "NARRATIO_INSPECT_NOTARIUS_ONE"} {
if !strings.Contains(show.String(), reference) {
t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String())
}
}
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") ||
!strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_TWO") {
t.Fatalf("diff omitted safe credential references: %q", diff.String())
}
if !strings.Contains(sources.String(), "notarius.credentials") {
t.Fatalf("sources omitted Notarius credential ownership: %q", sources.String())
}
if strings.Contains(removed.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") {
t.Fatalf("empty overlay did not remove Notarius credential: %q", removed.String())
}
}
func TestConfigValidateRejectsUnavailableNotariusCredential(t *testing.T) {
const envName = "NARRATIO_INSPECT_NOTARIUS_MISSING"
restoreEnvAfterTest(t, envName)
path := filepath.Join(t.TempDir(), "pipeline.yml")
mustWriteTestFile(t, path, "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n credentials: ["+envName+"]\n")
err := ConfigValidate(context.Background(), []string{"--config", path}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "secret.unset") {
t.Fatalf("ConfigValidate() error = %v, want unavailable credential failure", err)
}
}
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
examplesDir := filepath.Join("..", "..", "examples")
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
@@ -285,53 +467,45 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
t.Fatalf("example workspace stat = %v, want absent", err)
}
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} {
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil {
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources, ConfigArtifacts} {
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, io.Discard); err != nil {
t.Fatalf("inspection command %T error = %v", command, err)
}
}
var diff bytes.Buffer
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
if err := ConfigDiff(context.Background(), []string{"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
t.Fatalf("ConfigDiff() error = %v", err)
}
if !strings.Contains(diff.String(), "audita.model") {
t.Fatalf("split bundle profile diff = %q, want model change", diff.String())
if !strings.Contains(diff.String(), "audita.proposal_profile") {
t.Fatalf("split bundle overlay diff = %q, want proposal profile change", diff.String())
}
if _, err := os.Stat(workspacePath); !os.IsNotExist(err) {
t.Fatalf("inspection created example workspace: %v", err)
}
}
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
func writeInspectionOverlays(t *testing.T, pipelinePath, production, testing string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
overlaysDir := filepath.Join(filepath.Dir(pipelinePath), "overlays")
mustWriteTestFile(t, filepath.Join(overlaysDir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(overlaysDir, "testing.yml"), testing)
}
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
func TestConfigCommandsSelectOverlaysAndCampaigns(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
enableCommandTestProfiles(t, pipelinePath)
enableCommandTestOverlays(t, pipelinePath)
var defaultOut bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
t.Fatalf("default ConfigValidate() error = %v", err)
}
if !strings.Contains(defaultOut.String(), "profile=production") {
if !strings.Contains(defaultOut.String(), "overlay=none") {
t.Fatalf("default output = %q", defaultOut.String())
}
var explicitOut bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &explicitOut); err != nil {
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--overlay", "testing"}, &explicitOut); err != nil {
t.Fatalf("explicit ConfigShow() error = %v", err)
}
if !strings.Contains(explicitOut.String(), "language: fr") {
@@ -352,7 +526,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
func TestConfigCommandsExpandFamiliesAndRequireCampaign(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow} {
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigArtifacts} {
var out bytes.Buffer
err := command(context.Background(), []string{"--config", pipelinePath}, &out)
if err == nil || !strings.Contains(err.Error(), "artifact_families requires a campaign") {
@@ -398,8 +572,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [conf.yml]
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: conf.yml
workspace:
root: /one
whisperx:
@@ -407,7 +582,7 @@ whisperx:
`)
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
t.Fatalf("import diagnostic = %v", err)
}
}
@@ -415,9 +590,9 @@ whisperx:
func TestConfigSourcesReportsStableOwnership(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
enableCommandTestProfiles(t, pipelinePath)
enableCommandTestOverlays(t, pipelinePath)
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--overlay", "testing"}
var first, second bytes.Buffer
if err := ConfigSources(context.Background(), args, &first); err != nil {
t.Fatalf("first ConfigSources() error = %v", err)
@@ -432,13 +607,13 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
for _, want := range []string{
"root: ",
"imports: none",
"profile: name=testing selection=cli overlay=",
"overlay: name=testing selection=explicit path=",
"campaign: id=sample-campaign source=",
"party: source=",
"digest: ",
"path\trole\tsource\n",
"workspace.root\troot\t",
"whisperx.language\tprofile\t",
"whisperx.language\toverlay\t",
"derived.players\tparty\t",
"trim.enabled\tdefault\tdefault",
} {
@@ -473,18 +648,10 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
}
}
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
func TestConfigSourcesPreservesOverlayOwnershipThroughFamilyExpansion(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
mustWriteTestFile(t, profilePath, `scriptorium:
overlayPath := filepath.Join(filepath.Dir(pipelinePath), "overlays", "testing.yml")
mustWriteTestFile(t, overlayPath, `scriptorium:
artifact_families:
character_note:
enabled: true
@@ -492,45 +659,102 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
`)
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--overlay", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + overlayPath
if !strings.Contains(out.String(), want) {
t.Fatalf("sources output missing profile-generated ownership %q:\n%s", want, out.String())
t.Fatalf("sources output missing overlay-generated ownership %q:\n%s", want, out.String())
}
}
func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) {
func TestConfigSourcesAttributesInferredArtifactDependencies(t *testing.T) {
dir := t.TempDir()
pipelinePath := filepath.Join(dir, "pipeline.yml")
inputPath := filepath.Join(dir, "input-only.yml")
overlayPath := filepath.Join(dir, "overlays", "testing.yml")
mustWriteTestFile(t, pipelinePath, `configkit:
imports:
- path: input-only.yml
workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.test/transcribe
scriptorium:
artifacts:
first:
enabled: false
output_path: artifacts/first.md
second:
enabled: false
output_path: artifacts/second.md
mixed:
enabled: false
output_path: artifacts/mixed.md
depends_on: [first]
`)
mustWriteTestFile(t, inputPath, `scriptorium:
artifacts:
input_only:
enabled: false
output_path: artifacts/input-only.md
inputs:
prior:
source: narratio.artifact.first
`)
mustWriteTestFile(t, overlayPath, `scriptorium:
artifacts:
mixed:
inputs:
prior:
source: narratio.artifact.second
`)
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--overlay", "testing"}, &out); err != nil {
t.Fatal(err)
}
for _, want := range []string{
"scriptorium.artifacts.input_only.depends_on\timport\t" + inputPath,
"scriptorium.artifacts.mixed.depends_on\troot\t" + pipelinePath,
"scriptorium.artifacts.mixed.depends_on\toverlay\t" + overlayPath,
} {
if !strings.Contains(out.String(), want) {
t.Fatalf("sources output missing dependency contributor %q:\n%s", want, out.String())
}
}
if strings.Contains(out.String(), "scriptorium.artifacts.input_only.depends_on\tdefault\tdefault") {
t.Fatalf("inferred dependency retained default attribution:\n%s", out.String())
}
}
func TestConfigSourcesReportsRootImportOverlayAndDefaultOwnership(t *testing.T) {
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
importPath := filepath.Join(dir, "base.yml")
profilePath := filepath.Join(dir, "testing.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [base.yml]
default_profile: testing
profiles:
testing:
overlay: testing.yml
overlayPath := filepath.Join(dir, "overlays", "testing.yml")
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: base.yml
workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
mustWriteTestFile(t, importPath, "storage:\n backend: local\n")
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
mustWriteTestFile(t, overlayPath, "whisperx:\n language: fr\n")
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--overlay", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
output := out.String()
for _, want := range []string{
"import: " + importPath,
"profile: name=testing selection=default overlay=" + profilePath,
"overlay: name=testing selection=explicit path=" + overlayPath,
"workspace.root\troot\t" + rootPath,
"storage.backend\timport\t" + importPath,
"whisperx.language\tprofile\t" + profilePath,
"whisperx.language\toverlay\t" + overlayPath,
"trim.enabled\tdefault\tdefault",
} {
if !strings.Contains(output, want) {

View File

@@ -16,11 +16,12 @@ type configDiffFlags struct {
pipelinePath singletonStringFlag
campaignPath singletonStringFlag
campaignFilePath singletonStringFlag
secretsDir singletonStringFlag
}
type configDiffRequest struct {
leftProfile string
rightProfile string
leftOverlay string
rightOverlay string
flags configDiffFlags
}
@@ -36,11 +37,15 @@ type configDiffRecord struct {
Right string
}
// ConfigDiff compares two explicitly selected profiles through the same
// ConfigDiff compares two explicitly selected overlays through the same
// read-only configuration resolution boundary used by config validate, show,
// and sources. Differences describe normalized effective values rather than
// the layout of root, import, or overlay files.
func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
func ConfigDiff(ctx context.Context, args []string, out io.Writer) error {
return configDiffWithDependencies(ctx, args, out, commandDependencies{})
}
func configDiffWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
request, err := parseConfigDiffRequest(args, out)
if err != nil {
if errors.Is(err, flag.ErrHelp) {
@@ -48,7 +53,7 @@ func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
}
return err
}
resolved, err := resolveConfigDiff(request)
resolved, err := resolveConfigDiff(request, deps)
if err != nil {
return fmt.Errorf("config diff: %w", err)
}
@@ -95,20 +100,22 @@ func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
}
func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, error) {
profiles, flagArgs := splitConfigDiffArguments(args)
overlays, flagArgs := splitConfigDiffArguments(args)
fs := flag.NewFlagSet("config diff", flag.ContinueOnError)
fs.SetOutput(out)
var request configDiffRequest
request.flags.pipelinePath.name = "config"
request.flags.campaignPath.name = "campaign"
request.flags.campaignFilePath.name = "campaign-file"
request.flags.secretsDir.name = "secrets-dir"
fs.Var(&request.flags.pipelinePath, "config", "path to pipeline.yml (optional; defaults searched)")
fs.Var(&request.flags.campaignPath, "campaign", "campaign ID")
fs.Var(&request.flags.campaignFilePath, "campaign-file", "path to campaign.yml")
fs.Var(&request.flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
fs.Usage = func() {
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]")
fmt.Fprintln(out, "Usage: narratio config diff <left-overlay> <right-overlay> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <path>]")
fmt.Fprintln(out)
fmt.Fprintln(out, "Compares fully resolved profiles without a session, workspace, manifest, adapters, or external services.")
fmt.Fprintln(out, "Compares fully resolved overlays without a session, workspace, manifest, adapters, or external services.")
fmt.Fprintln(out)
fmt.Fprintln(out, "Flags:")
fs.PrintDefaults()
@@ -116,27 +123,30 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
if err := fs.Parse(flagArgs); err != nil {
return configDiffRequest{}, fmt.Errorf("config diff: invalid flags: %w", err)
}
profiles = append(profiles, fs.Args()...)
if len(profiles) != 2 || strings.TrimSpace(profiles[0]) == "" || strings.TrimSpace(profiles[1]) == "" {
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty profile names are required")
overlays = append(overlays, fs.Args()...)
if len(overlays) != 2 || strings.TrimSpace(overlays[0]) == "" || strings.TrimSpace(overlays[1]) == "" {
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty overlay names are required")
}
if overlays[0] == overlays[1] {
return configDiffRequest{}, fmt.Errorf("config diff: overlay names must differ")
}
if request.flags.campaignPath.set && request.flags.campaignFilePath.set {
return configDiffRequest{}, fmt.Errorf("config diff: --campaign and --campaign-file are mutually exclusive")
}
request.leftProfile = profiles[0]
request.rightProfile = profiles[1]
request.leftOverlay = overlays[0]
request.rightOverlay = overlays[1]
return request, nil
}
// splitConfigDiffArguments accepts the documented positional-first syntax and
// also permits flags before or between profile names. Values belonging to the
// also permits flags before or between overlay names. Values belonging to the
// supported string flags stay with their flag so they are not mistaken for
// profile names.
func splitConfigDiffArguments(args []string) (profiles, flagArgs []string) {
// overlay names.
func splitConfigDiffArguments(args []string) (overlays, flagArgs []string) {
for index := 0; index < len(args); index++ {
argument := args[index]
if !strings.HasPrefix(argument, "-") || argument == "-" {
profiles = append(profiles, argument)
overlays = append(overlays, argument)
continue
}
flagArgs = append(flagArgs, argument)
@@ -151,26 +161,27 @@ func splitConfigDiffArguments(args []string) (profiles, flagArgs []string) {
index++
flagArgs = append(flagArgs, args[index])
}
return profiles, flagArgs
return overlays, flagArgs
}
func configDiffStringFlag(name string) bool {
switch name {
case "config", "campaign", "campaign-file", "profile":
case "config", "campaign", "campaign-file", "secrets-dir":
return true
default:
return false
}
}
func resolveConfigDiff(request configDiffRequest) (*configDiffResolved, error) {
pipelinePath, err := resolvePipelineConfigPath(request.flags.pipelinePath.value)
func resolveConfigDiff(request configDiffRequest, deps commandDependencies) (*configDiffResolved, error) {
deps = normalizeCommandDependencies(deps)
pipelinePath, leftPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Overlay: &request.leftOverlay}, request.flags.secretsDir.pointer(), deps)
if err != nil {
return nil, err
return nil, fmt.Errorf("load left pipeline overlay: %w", err)
}
leftPipeline, rightPipeline, err := config.LoadPipelineProfilePair(pipelinePath, request.leftProfile, request.rightProfile)
_, rightPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Overlay: &request.rightOverlay}, request.flags.secretsDir.pointer(), deps)
if err != nil {
return nil, err
return nil, fmt.Errorf("load right pipeline overlay: %w", err)
}
needsCampaign := pipelineHasArtifactFamilies(leftPipeline) || pipelineHasArtifactFamilies(rightPipeline)
hasCampaignSelection := request.flags.campaignPath.set || request.flags.campaignFilePath.set
@@ -248,7 +259,7 @@ func resolveSharedDiffCampaignPath(left, right *config.PipelineConfig, flags con
return "", rightErr
}
if leftPath != rightPath {
return "", fmt.Errorf("profile selections resolve different campaign files; specify --campaign-file to compare one campaign")
return "", fmt.Errorf("overlay selections resolve different campaign files; specify --campaign-file to compare one campaign")
}
return leftPath, nil
}

View File

@@ -34,7 +34,7 @@ func retainedCommandConfig(cfg *config.Config) *commandConfig {
return &commandConfig{Config: cfg}
}
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *commandConfig, err error) {
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (loaded *commandConfig, err error) {
deps = normalizeCommandDependencies(deps)
var cleanup func() error
defer func() {
@@ -46,7 +46,7 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
}
}()
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, deps)
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Overlay: sessionOpts.Overlay}, secretsDir, deps)
if err != nil {
return nil, err
}
@@ -135,9 +135,9 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
return loaded, nil
}
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, deps commandDependencies) (*pipelineCampaignConfig, error) {
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) {
deps = normalizeCommandDependencies(deps)
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, deps)
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps)
if err != nil {
return nil, err
}
@@ -161,12 +161,19 @@ func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, camp
return &loaded, nil
}
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, deps commandDependencies) (string, *config.PipelineConfig, error) {
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) {
deps = normalizeCommandDependencies(deps)
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
resolvedPipelinePath, err := deps.resolvePipelineConfigPath(pipelineFlag)
if err != nil {
return "", nil, err
}
if !deps.secretsBootstrap.selectionSet {
deps.secretsBootstrap.selection = secretsDir
deps.secretsBootstrap.selectionSet = true
}
if err := deps.secretsBootstrap.load(resolvedPipelinePath, deps.loadSecretsDir, deps.diagnostics); err != nil {
return "", nil, err
}
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
if err != nil {
return "", nil, err

View File

@@ -35,7 +35,7 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
},
}
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
if err != nil {
t.Fatalf("loadCommandConfig() error = %v", err)
}

View File

@@ -14,13 +14,13 @@ func effectiveConfigProvenance(cfg *config.Config) *manifest.EffectiveConfigProv
provenance := &manifest.EffectiveConfigProvenance{
EffectiveConfigDigest: config.EffectivePipelineDigest(cfg.Pipeline),
}
if selected, ok := config.SelectedPipelineProfile(cfg.Pipeline); ok {
provenance.SelectedProfile = &manifest.SelectedProfileProvenance{
if selected, ok := config.SelectedPipelineOverlay(cfg.Pipeline); ok {
provenance.SelectedOverlay = &manifest.SelectedOverlayProvenance{
Name: selected.Name,
Source: selected.Source,
}
}
if provenance.SelectedProfile == nil && provenance.EffectiveConfigDigest == "" {
if provenance.SelectedOverlay == nil && provenance.EffectiveConfigDigest == "" {
return nil
}
return provenance
@@ -35,30 +35,30 @@ func applyEffectiveConfigProvenance(m *manifest.Manifest, cfg *config.Config) {
func effectiveConfigSummary(cfg *config.Config) string {
provenance := effectiveConfigProvenance(cfg)
if provenance == nil {
return "profile=none digest=unavailable"
return "overlay=none digest=unavailable"
}
profile := "none"
if provenance.SelectedProfile != nil {
profile = provenance.SelectedProfile.Name + " (" + provenance.SelectedProfile.Source + ")"
overlay := "none"
if provenance.SelectedOverlay != nil {
overlay = provenance.SelectedOverlay.Name + " (" + provenance.SelectedOverlay.Source + ")"
}
digest := provenance.EffectiveConfigDigest
if digest == "" {
digest = "unavailable"
}
return fmt.Sprintf("profile=%s digest=%s", profile, digest)
return fmt.Sprintf("overlay=%s digest=%s", overlay, digest)
}
func persistedEffectiveConfigSummary(provenance *manifest.EffectiveConfigProvenance) string {
if provenance == nil {
return "profile=none digest=unavailable"
return "overlay=none digest=unavailable"
}
profile := "none"
if provenance.SelectedProfile != nil {
profile = provenance.SelectedProfile.Name + " (" + provenance.SelectedProfile.Source + ")"
overlay := "none"
if provenance.SelectedOverlay != nil {
overlay = provenance.SelectedOverlay.Name + " (" + provenance.SelectedOverlay.Source + ")"
}
digest := provenance.EffectiveConfigDigest
if digest == "" {
digest = "unavailable"
}
return fmt.Sprintf("profile=%s digest=%s", profile, digest)
return fmt.Sprintf("overlay=%s digest=%s", overlay, digest)
}

View File

@@ -7,11 +7,13 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
@@ -181,12 +183,16 @@ func TestExtractLifecyclePrepareBindsVerifiedReferenceSnapshots(t *testing.T) {
for index, expected := range want {
binding := request.References[index]
canonical := filepath.Join(paths.InputsDir, expected.filename)
snapshot := filepath.Join(
artifacts.SessionRunNotariusReferencesDirForCampaign(
cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, loaded.RunID,
),
expected.filename,
descriptor, ok := artifactpolicy.DescribePreparedInputSource(expected.sourceID)
if !ok {
t.Fatalf("DescribePreparedInputSource(%q) ok = false", expected.sourceID)
}
snapshot, err := artifacts.SessionNotariusReferenceSnapshotPath(
paths, lifecycleInputChecksum(t, loaded, descriptor.ManifestKind), expected.filename,
)
if err != nil {
t.Fatalf("SessionNotariusReferenceSnapshotPath(%q) error = %v", expected.sourceID, err)
}
if binding.Selector != expected.selector || binding.Path != snapshot || binding.Path == canonical || binding.Path == originalPaths[expected.sourceID] {
t.Fatalf("reference[%d] = %#v, want selector %q snapshot %q and not prepared/source paths", index, binding, expected.selector, snapshot)
}
@@ -393,6 +399,140 @@ func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *t
}
}
func TestExtractLifecycleRetryReusesStableSnapshotsAndChangedReferenceSelectsNewPath(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
configureLifecycleReferences(t, cfg)
runner.failuresRemaining = 1
plan := prepareExtractLifecyclePlan(t)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
t.Fatal("first executeStages() error = nil, want extraction failure")
}
if len(runner.requests) != 1 {
t.Fatalf("failed requests = %d, want 1", len(runner.requests))
}
firstPaths := lifecycleReferencePaths(runner.requests[0])
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
if len(runner.requests) != 2 {
t.Fatalf("retry requests = %d, want 2", len(runner.requests))
}
if secondPaths := lifecycleReferencePaths(runner.requests[1]); !reflect.DeepEqual(secondPaths, firstPaths) {
t.Fatalf("snapshot paths changed on retry:\nfirst %#v\nsecond %#v", firstPaths, secondPaths)
}
partyDocument, err := config.ParseParty([]byte(`schema_version: narratio.party.v1
characters:
changed:
player: {name: Eric}
character:
name: Changed
classes: [{name: wizard}]
`))
if err != nil {
t.Fatalf("ParseParty(changed party) error = %v", err)
}
cfg.Party.Canonical = partyDocument
persisted := loadLifecycleManifest(t, cfg)
prepare, err := stage.Select("prepare")
if err != nil {
t.Fatalf("stage.Select(prepare) error = %v", err)
}
if _, err := prepare.Run(context.Background(), env, persisted); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), persisted); err != nil {
t.Fatalf("Save(reprepared manifest) error = %v", err)
}
extractPlan, _ := BuildSingleStagePlan("extract")
if _, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env}); err != nil {
t.Fatalf("changed-reference executeStages() error = %v", err)
}
if len(runner.requests) != 3 {
t.Fatalf("changed-reference requests = %d, want 3", len(runner.requests))
}
thirdPaths := lifecycleReferencePaths(runner.requests[2])
if thirdPaths["party"] == firstPaths["party"] {
t.Fatalf("changed party retained snapshot path %q", thirdPaths["party"])
}
if thirdPaths["players"] == firstPaths["players"] {
t.Fatalf("changed party retained derived players snapshot path %q", thirdPaths["players"])
}
for selector, firstPath := range firstPaths {
if selector != "party" && selector != "players" && thirdPaths[selector] != firstPath {
t.Fatalf("unchanged %s snapshot path = %q, want %q", selector, thirdPaths[selector], firstPath)
}
if _, err := os.Stat(firstPath); err != nil {
t.Fatalf("old %s snapshot %q was not retained: %v", selector, firstPath, err)
}
}
}
func TestExtractLifecycleCacheControlsAndSnapshotLossDoNotInvalidateCompleteResult(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
configureLifecycleReferences(t, cfg)
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
cacheRoot := artifacts.SessionNotariusReferenceCacheDir(paths)
if err := os.RemoveAll(cacheRoot); err != nil {
t.Fatalf("RemoveAll(snapshot cache) error = %v", err)
}
cfg.Pipeline.Notarius.Resume = true
cfg.Pipeline.Notarius.ChunkCache = "bypass"
extractPlan, _ := BuildSingleStagePlan("extract")
summary, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env})
if err != nil {
t.Fatalf("resume executeStages() error = %v", err)
}
if len(summary.Executed) != 0 || !reflect.DeepEqual(summary.Skipped, []string{"extract"}) || len(runner.requests) != 1 {
t.Fatalf("resume summary = %#v requests=%d", summary, len(runner.requests))
}
if _, err := os.Stat(cacheRoot); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("skipped extraction recreated snapshot cache: %v", err)
}
}
func TestAnalyzeOnlyLifecyclePreservesExtractionWithoutInvokingNotarius(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
extractPlan, _ := BuildSingleStagePlan("extract")
if _, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env}); err != nil {
t.Fatalf("extract executeStages() error = %v", err)
}
before := loadLifecycleManifest(t, cfg).Stages["extract"]
beforeJSON, _ := json.Marshal(before)
analyzeRuns := 0
if _, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "analyze", runs: &analyzeRuns}}, RunOptions{Env: env, Force: true}); err != nil {
t.Fatalf("analyze-only executeStages() error = %v", err)
}
after := loadLifecycleManifest(t, cfg).Stages["extract"]
afterJSON, _ := json.Marshal(after)
if analyzeRuns != 1 || len(runner.requests) != 1 || string(afterJSON) != string(beforeJSON) {
t.Fatalf("analyze-only lifecycle changed extraction: analyze=%d requests=%d\nbefore=%s\nafter=%s", analyzeRuns, len(runner.requests), beforeJSON, afterJSON)
}
for _, output := range after.Outputs {
if _, err := os.Stat(output.LocalPath); err != nil {
t.Fatalf("analyze-only lifecycle removed extract output %q: %v", output.LocalPath, err)
}
}
}
func TestForcedArtifactRegenerationUsesOrdinaryExtractAndAnalyzePath(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
cfg.Pipeline.Notarius.Resume = true
cfg.Pipeline.Notarius.ChunkCache = "bypass"
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true, SelectedArtifacts: []string{"session_recap"}}); err != nil {
t.Fatalf("forced artifact regeneration error = %v", err)
}
if len(runner.requests) != 1 || runner.requests[0].Resume || runner.requests[0].ChunkCache != "refresh" || analyzeRuns != 1 {
t.Fatalf("forced path request=%#v analyze=%d", runner.requests, analyzeRuns)
}
}
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
cfg, env, _ := extractionLifecycleFixture(t, false)
markLifecycleStageSucceeded(t, cfg, "analyze")
@@ -485,6 +625,273 @@ func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
}
}
func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) {
for _, test := range []struct {
name string
before func(*testing.T) *configkit.Secret
after func(*testing.T) *configkit.Secret
}{
{
name: "added",
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ADDED_KEY", "added-value")
},
},
{
name: "removed",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_REMOVED_KEY", "removed-value")
},
},
{
name: "renamed",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_FIRST_KEY", "first-value")
},
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_SECOND_KEY", "second-value")
},
},
{
name: "value rotated",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "first-value")
},
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "second-value")
},
},
} {
t.Run(test.name, func(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
if test.before != nil {
cfg.Pipeline.Notarius.Credentials = []configkit.Secret{*test.before(t)}
}
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
cfg.Pipeline.Notarius.Credentials = nil
if test.after != nil {
cfg.Pipeline.Notarius.Credentials = []configkit.Secret{*test.after(t)}
}
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("resume executeStages() error = %v", err)
}
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
t.Fatalf("credential-only change reexecuted work: summary=%#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
}
loaded := loadLifecycleManifest(t, cfg)
if loaded.Stages["extract"].Status != manifest.StatusSucceeded || loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
t.Fatalf("credential-only change invalidated stages: extract=%q analyze=%q", loaded.Stages["extract"].Status, loaded.Stages["analyze"].Status)
}
})
}
}
func TestExtractLifecycleConfigProfileChangesRerunExtractionAndDependents(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
selected := "inexpensive"
cfg.Pipeline.Notarius.ConfigProfile = &selected
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != selected || analyzeRuns != 1 {
t.Fatalf("initial requests=%#v analyze=%d, want selected extraction and analysis", runner.requests, analyzeRuns)
}
reused, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("reuse executeStages() error = %v", err)
}
if len(reused.Executed) != 0 || len(runner.requests) != 1 || analyzeRuns != 1 {
t.Fatalf("same-profile reuse summary=%#v requests=%d analyze=%d", reused, len(runner.requests), analyzeRuns)
}
markLifecycleStageSucceeded(t, cfg, "render")
markLifecycleStageSucceeded(t, cfg, "publish")
before := loadLifecycleManifest(t, cfg)
trimBefore, err := json.Marshal(before.Stages["trim"])
if err != nil {
t.Fatal(err)
}
changed := "experimental"
cfg.Pipeline.Notarius.ConfigProfile = &changed
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("changed-profile executeStages() error = %v", err)
}
if len(rerun.Executed) != 2 || len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed || analyzeRuns != 2 {
t.Fatalf("changed-profile summary=%#v requests=%#v analyze=%d", rerun, runner.requests, analyzeRuns)
}
after := loadLifecycleManifest(t, cfg)
trimAfter, err := json.Marshal(after.Stages["trim"])
if err != nil {
t.Fatal(err)
}
if string(trimAfter) != string(trimBefore) {
t.Fatalf("profile change altered upstream trim record:\nbefore=%s\nafter=%s", trimBefore, trimAfter)
}
if after.Stages["render"].Status != manifest.StatusSucceeded || after.Stages["analyze"].Status != manifest.StatusSucceeded || after.Stages["publish"].Status != manifest.StatusStale {
t.Fatalf("changed-profile lifecycle render=%#v analyze=%#v publish=%#v", after.Stages["render"], after.Stages["analyze"], after.Stages["publish"])
}
cfg.Pipeline.Notarius.ConfigProfile = nil
returned, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("return-to-base executeStages() error = %v", err)
}
if len(returned.Executed) != 2 || len(runner.requests) != 3 || runner.requests[2].ConfigProfile != nil || analyzeRuns != 3 {
t.Fatalf("return-to-base summary=%#v requests=%#v analyze=%d", returned, runner.requests, analyzeRuns)
}
final := loadLifecycleManifest(t, cfg)
if final.Stages["trim"].Status != manifest.StatusSucceeded || final.Stages["render"].Status != manifest.StatusSucceeded {
t.Fatalf("return-to-base invalidated upstream siblings: trim=%#v render=%#v", final.Stages["trim"], final.Stages["render"])
}
}
func TestExtractLifecycleConfigProfileFailureClearsCurrentResultAndDependents(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
selected := "inexpensive"
cfg.Pipeline.Notarius.ConfigProfile = &selected
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
markLifecycleStageSucceeded(t, cfg, "render")
markLifecycleStageSucceeded(t, cfg, "publish")
changed := "missing-profile"
cfg.Pipeline.Notarius.ConfigProfile = &changed
runner.failuresRemaining = 1
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
t.Fatal("changed-profile executeStages() error = nil, want downstream failure")
}
failed := loadLifecycleManifest(t, cfg)
extract := failed.Stages["extract"]
if extract.Status != manifest.StatusFailed || len(extract.Outputs) != 0 || len(extract.Logs) != 0 || len(extract.GeneratedConfigs) != 0 || len(extract.Metadata) != 0 {
t.Fatalf("failed changed-profile extraction advertised prior success: %#v", extract)
}
if failed.Stages["trim"].Status != manifest.StatusSucceeded || failed.Stages["render"].Status != manifest.StatusSucceeded ||
failed.Stages["analyze"].Status != manifest.StatusStale || failed.Stages["publish"].Status != manifest.StatusStale || analyzeRuns != 1 {
t.Fatalf("failed changed-profile lifecycle trim=%#v render=%#v analyze=%#v publish=%#v analyzeRuns=%d",
failed.Stages["trim"], failed.Stages["render"], failed.Stages["analyze"], failed.Stages["publish"], analyzeRuns)
}
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed {
t.Fatalf("failed request = %#v, want changed selector", runner.requests)
}
}
func TestExtractLifecycleRealConfigProfilesSelectNotariusExplicitly(t *testing.T) {
seed, env, runner := extractionLifecycleFixture(t, true)
workspace := seed.Pipeline.Workspace.Root
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspace)
pipelineFile, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
if err != nil {
t.Fatal(err)
}
_, writeErr := pipelineFile.WriteString(`notarius:
enabled: true
binary: ` + filepath.Join(workspace, "notarius") + `
config_path: notarius.yml
pipeline_id: dnd-session
timeout: 45m
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
`)
closeErr := pipelineFile.Close()
if writeErr != nil || closeErr != nil {
t.Fatalf("write Notarius configuration: %v / %v", writeErr, closeErr)
}
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("pipelines: {}\n"), 0o644); err != nil {
t.Fatal(err)
}
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "overlays")
if err := os.MkdirAll(profilesDir, 0o755); err != nil {
t.Fatal(err)
}
for name, contents := range map[string]string{
"testing": "whisperx:\n language: fr\nnotarius:\n config_profile: downstream-economy\n",
"narratio-only": "whisperx:\n language: de\n",
"narratio-only-two": "whisperx:\n language: es\n",
} {
if err := os.WriteFile(filepath.Join(profilesDir, name+".yml"), []byte(contents), 0o644); err != nil {
t.Fatal(err)
}
}
load := func(profile string) *config.Config {
t.Helper()
loaded, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{Overlay: &profile})
if err != nil {
t.Fatalf("load profile %q: %v", profile, err)
}
return loaded
}
plan, err := BuildSingleStagePlan("extract")
if err != nil {
t.Fatal(err)
}
testingConfig := load("testing")
runner.cfg = testingConfig.Pipeline.Notarius
if _, err := executeStages(context.Background(), testingConfig, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("execute testing profile: %v", err)
}
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != "downstream-economy" {
t.Fatalf("testing profile request = %#v, want explicit differently named downstream selector", runner.requests)
}
narratioOnly := load("narratio-only")
runner.cfg = narratioOnly.Pipeline.Notarius
if _, err := executeStages(context.Background(), narratioOnly, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("execute Narratio-only profile: %v", err)
}
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile != nil {
t.Fatalf("Narratio-only profile request = %#v, want no automatic downstream selector", runner.requests)
}
narratioOnlyTwo := load("narratio-only-two")
runner.cfg = narratioOnlyTwo.Pipeline.Notarius
reused, err := executeStages(context.Background(), narratioOnlyTwo, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("execute second Narratio-only profile: %v", err)
}
if len(reused.Executed) != 0 || len(runner.requests) != 2 {
t.Fatalf("Narratio profile label/effective digest broadened invalidation: summary=%#v requests=%#v", reused, runner.requests)
}
}
func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) {
const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY"
const canary = "notarius-persisted-credential-canary"
cfg, env, _ := extractionLifecycleFixture(t, true)
cfg.Pipeline.Notarius.Credentials = []configkit.Secret{*loadLifecycleNotariusCredential(t, name, canary)}
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
for _, path := range []string{paths.ManifestPath, filepath.Join(paths.InputsDir, "pipeline.resolved.yml")} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
if strings.Contains(string(data), canary) {
t.Fatalf("persisted Narratio file %q contains the credential value", path)
}
}
}
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
analyzeRuns := 0
@@ -696,6 +1103,14 @@ func lifecycleInputChecksum(t *testing.T, m *manifest.Manifest, kind string) str
return ""
}
func lifecycleReferencePaths(request notarius.RunRequest) map[string]string {
paths := make(map[string]string, len(request.References))
for _, reference := range request.References {
paths[reference.Selector] = reference.Path
}
return paths
}
func lifecycleDndOutputs() map[string]config.NotariusOutputConfig {
return map[string]config.NotariusOutputConfig{
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
@@ -761,3 +1176,17 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
return cfg, &stage.Env{Notarius: runner}, runner
}
func loadLifecycleNotariusCredential(t *testing.T, name, value string) *configkit.Secret {
t.Helper()
t.Setenv(name, value)
path := filepath.Join(t.TempDir(), "pipeline.yml")
if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n credentials: ["+name+"]\n"), 0o644); err != nil {
t.Fatal(err)
}
pipeline, err := config.LoadPipeline(path)
if err != nil {
t.Fatal(err)
}
return &pipeline.Notarius.Credentials[0]
}

View File

@@ -11,9 +11,6 @@ import (
func newCommandObjectStoreWithDependencies(ctx context.Context, cfg *config.Config, logger *slog.Logger, deps commandDependencies) (storage.ObjectStore, error) {
deps = normalizeCommandDependencies(deps)
if _, err := loadSecretsFromConfigWithDependencies(cfg, logger, deps); err != nil {
return nil, fmt.Errorf("load secrets from files: %w", err)
}
return initializeObjectStoreBackend(ctx, cfg, deps.newObjectStore)
}

View File

@@ -3,8 +3,6 @@ package app
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
@@ -12,89 +10,8 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
fake := &storage.FakeBackend{}
called := false
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
return nil, errors.New("access key was not loaded before object store init")
}
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
return nil, errors.New("secret key was not loaded before object store init")
}
return fake, nil
}}
store, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
if err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
if store != fake {
t.Fatalf("store = %#v, want fake backend", store)
}
if !called {
t.Fatal("object store factory was not called")
}
}
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
t.Setenv(accessKeyEnv, "existing-key-id")
t.Setenv(secretKeyEnv, "existing-secret")
secretsDir := t.TempDir()
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
cfg := commandObjectStoreTestConfig(secretsDir)
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
return nil, errors.New("existing access key was overwritten")
}
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
return nil, errors.New("existing secret key was overwritten")
}
return &storage.FakeBackend{}, nil
}}
if _, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps); err != nil {
t.Fatalf("newCommandObjectStore() error = %v", err)
}
}
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
called := false
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
called = true
return &storage.FakeBackend{}, nil
}}
_, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
if err == nil {
t.Fatal("expected error, got nil")
}
if called {
t.Fatal("object store factory was called after secret load failure")
}
if !strings.Contains(err.Error(), "load secrets from files") {
t.Fatalf("error = %q, want secret loading context", err.Error())
}
}
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
cfg := commandObjectStoreTestConfig("")
cfg := commandObjectStoreTestConfig()
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
return nil, errors.New("factory boom")
}}
@@ -108,21 +25,17 @@ func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
}
}
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
cfg := &config.Config{
func commandObjectStoreTestConfig() *config.Config {
return &config.Config{
Pipeline: &config.PipelineConfig{
Storage: config.StorageConfig{
Backend: "s3",
S3: &config.StorageS3Config{
Bucket: "test-bucket",
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
Bucket: "test-bucket",
AccessKeyID: testSecretReference("NARRATIO_TEST_COMMAND_STORE_KEY_ID"),
SecretAccessKey: testSecretReference("NARRATIO_TEST_COMMAND_STORE_SECRET"),
},
},
},
}
if strings.TrimSpace(secretsDir) != "" {
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
}
return cfg
}

View File

@@ -23,7 +23,8 @@ type commonConfigFlags struct {
sessionPath string
sessionID string
previousSessionID string
profile singletonStringFlag
overlay singletonStringFlag
secretsDir singletonStringFlag
}
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
@@ -33,15 +34,17 @@ func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier")
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
flags.profile.name = "profile"
fs.Var(&flags.profile, "profile", "named pipeline profile")
flags.overlay.name = "overlay"
fs.Var(&flags.overlay, "overlay", "named pipeline overlay")
flags.secretsDir.name = "secrets-dir"
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
}
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
return config.SessionLoadOptions{
SessionID: f.sessionID,
PreviousSessionID: f.previousSessionID,
Profile: f.profile.pointer(),
Overlay: f.overlay.pointer(),
}
}
@@ -112,7 +115,7 @@ func artifactsWithDependencies(ctx context.Context, args []string, out io.Writer
func loadHelperContextWithDependencies(ctx context.Context, flags commonConfigFlags, needStore bool, deps commandDependencies) (*config.Config, storage.ObjectStore, []artifacts.RemoteLock, *manifest.Manifest, func(), error) {
deps = normalizeCommandDependencies(deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
if err != nil {
return nil, nil, nil, nil, nil, err
}

View File

@@ -164,19 +164,19 @@ func TestExecuteSessionInitRequiresSessionID(t *testing.T) {
}
func TestExecuteSessionInitMissingDefaultConfigReportsSearchedPaths(t *testing.T) {
origPipelineDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
config.DefaultPipelineConfigSearchPaths = []string{filepath.Join(t.TempDir(), "missing-pipeline.yml")}
t.Cleanup(func() {
config.DefaultPipelineConfigSearchPaths = origPipelineDefaults
})
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := Execute([]string{"session", "init", "2026-06-07", "--remote"}, &stdout, &stderr)
code := executeWithDependencies([]string{"session", "init", "2026-06-07", "--remote"}, &stdout, &stderr, deps, executor)
if code == 0 {
t.Fatal("exit code = 0, want non-zero")
}
if !strings.Contains(stderr.String(), "session init: no pipeline config path provided and no default pipeline config found; searched:") {
if stdout.Len() != 0 {
t.Fatalf("stdout = %q, want empty", stdout.String())
}
if got := stderr.String(); !strings.Contains(got, "session init: no pipeline config path provided") ||
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
t.Fatalf("stderr = %q, want default pipeline searched-path error", stderr.String())
}
}

View File

@@ -131,7 +131,7 @@ func locksRemoveWithDependencies(ctx context.Context, args []string, out io.Writ
func loadLockCommandContextWithDependencies(ctx context.Context, flags commonConfigFlags, deps commandDependencies) (*config.Config, storage.ObjectStore, string, func(), error) {
deps = normalizeCommandDependencies(deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
if err != nil {
return nil, nil, "", nil, err
}

View File

@@ -27,7 +27,8 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
fs.SetOutput(io.Discard)
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
var profile singletonStringFlag
var overlay singletonStringFlag
var secretsDir singletonStringFlag
var remote, force bool
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
@@ -39,8 +40,10 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
fs.StringVar(&output, "output", "", "local output session.yml path")
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
profile.name = "profile"
fs.Var(&profile, "profile", "named pipeline profile")
overlay.name = "overlay"
fs.Var(&overlay, "overlay", "named pipeline overlay")
secretsDir.name = "secrets-dir"
fs.Var(&secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
fs.BoolVar(&force, "force", false, "overwrite existing target")
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
@@ -56,7 +59,7 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
}
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, deps)
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Overlay: overlay.pointer()}, secretsDir.pointer(), deps)
if err != nil {
return fmt.Errorf("session init: %w", err)
}

View File

@@ -30,7 +30,7 @@ func sessionValidateWithDependencies(ctx context.Context, args []string, out io.
}
findings := []finding{}
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
if err != nil {
findings = append(findings, errorFinding("config", err.Error()))
return renderFindings(out, "", "", findings)

Some files were not shown because too many files have changed in this diff Show More