Compare commits
145 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 442469b9f5 | |||
| c45e042462 | |||
| 207da1b6ef | |||
| 9a1609435e | |||
| 13b83d7ee2 | |||
| a0312ed034 | |||
| 8ff74e1251 | |||
| 1b480d4ed5 | |||
| e13d87a077 | |||
| 5202f16972 | |||
| 5086596f94 | |||
| f7fc64db10 | |||
| 6b0778b420 | |||
| 5b92b3e790 | |||
| cc43906ae9 | |||
| dcaa2ad9c6 | |||
| 9d68cfcf2f | |||
| 2d346733d6 | |||
| cd75bf6d5a | |||
| e08de98ae6 | |||
| df213bc2ba | |||
| 869e68e834 | |||
| ed4623bc41 | |||
| 3ce198197d | |||
| 0ac8072a9a | |||
| 3ec54b8c72 | |||
| 47b3e5b55b | |||
| 99d1a0b926 | |||
| 884a62e8a6 | |||
| f7a40afc7a | |||
| a4f37ff427 | |||
| 2b97ef3722 | |||
| 63a86c43f4 | |||
| 92432232ef | |||
| d31ba5a410 | |||
| f0225e937c | |||
| 9e40c729e1 | |||
| 0e6ba414b3 | |||
| 10a2e89965 | |||
| f8e3e651e6 | |||
| 1558c9ccae | |||
| b68435128b | |||
| 5fb454a714 | |||
| 79cf9f1139 | |||
| 9bb203476d | |||
| 0726786016 | |||
| 204caf9d24 | |||
| 85663f5bc9 | |||
| f51b9bb046 | |||
| dd905c4daa | |||
| c3307ab5fb | |||
| 1484f51f48 | |||
| 51d21676c1 | |||
| c2806a1615 | |||
| 078f661e3a | |||
| adc60aa588 | |||
| 8d9eb5cc40 | |||
| 563acce012 | |||
| 49c359feb8 | |||
| 38202d4472 | |||
| 6d07dafc9f | |||
| 6c66e18c4b | |||
| 4048996652 | |||
| 470450e509 | |||
| 7e78c22fec | |||
| 610eaefb27 | |||
| 4bc33773fa | |||
| b74e31e1cc | |||
| 2f1e4407db | |||
| 4ccb4a73ca | |||
| 145baac393 | |||
| be59bff690 | |||
| 14b4fee214 | |||
| 7f84fc5d1e | |||
| a8b47f6185 | |||
| 5b62a8d60d | |||
| 0d4d71b9dc | |||
| 979d8049b1 | |||
| abd405015b | |||
| fddf78dc80 | |||
| 3136c26d4e | |||
| 4b84b4223d | |||
| 7a0e2cb11b | |||
| 74ab9218ff | |||
| da99857fd0 | |||
| a520cfdcc8 | |||
| 7febccf550 | |||
| 74977b4064 | |||
| 172e2d6a4f | |||
| 5c520dc0d4 | |||
| 6d18a3126e | |||
| 1e222a4db3 | |||
| 6a15052d77 | |||
| 2e69c14898 | |||
| dbda2a0161 | |||
| 2ee64f2eb1 | |||
| 8afe9427ca | |||
| e0f19833a1 | |||
| ca1b694aa9 | |||
| 9331373245 | |||
| ec072dc163 | |||
| f7ce1dcd56 | |||
| df275874e3 | |||
| 1da34ab5cd | |||
| c0fbc14f2d | |||
| 2d20900cbf | |||
| 7997f81f47 | |||
| 6d748d65d3 | |||
| 49324d3f35 | |||
| c98b4ef4ef | |||
| c8bae3a4a3 | |||
| ccbd7ee820 | |||
| e9c4873030 | |||
| 57719b7d81 | |||
| 62b1f9b0ae | |||
| 6c7a45d8d0 | |||
| 82d75e7beb | |||
| 61095e69ce | |||
| 2cfca710ec | |||
| 9679117e94 | |||
| d67a0389e0 | |||
| 9f84446509 | |||
| ec9b0e919d | |||
| 9bb7c1f4e3 | |||
| 3c092fbe34 | |||
| 3a7e2e5175 | |||
| 45b27ad562 | |||
| 1e418d6929 | |||
| 39b2665d8b | |||
| ba858cc997 | |||
| 0d57422b61 | |||
| b7d989f61f | |||
| 6813340f51 | |||
| eb788f17b8 | |||
| ebd3f415d3 | |||
| f15d8d8ce0 | |||
| 6c7bfd5e31 | |||
| 7c782a2603 | |||
| f1eef5b5e9 | |||
| f14583321d | |||
| 04283e99d4 | |||
| 7284228bec | |||
| da74d7ad18 | |||
| af5fbc1c4f | |||
| 5e4d2016e8 |
@@ -3,12 +3,12 @@ when:
|
||||
|
||||
steps:
|
||||
validate-release:
|
||||
image: golang:1.25.5
|
||||
image: golang:1.26.7
|
||||
commands:
|
||||
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
|
||||
|
||||
build-release-assets:
|
||||
image: golang:1.25.5
|
||||
image: golang:1.26.7
|
||||
depends_on:
|
||||
- validate-release
|
||||
commands:
|
||||
|
||||
@@ -3,6 +3,6 @@ when:
|
||||
|
||||
steps:
|
||||
shuffled-race-tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test -race -shuffle=on -count=3 ./...
|
||||
|
||||
@@ -3,44 +3,44 @@ when:
|
||||
|
||||
steps:
|
||||
tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test ./...
|
||||
|
||||
race-tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test -race ./...
|
||||
|
||||
static-analysis:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go vet ./...
|
||||
|
||||
build:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go build ./...
|
||||
|
||||
documentation-and-examples:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test ./internal/doccheck
|
||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
|
||||
cross-build:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
output_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$output_dir"' EXIT
|
||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||
for target in linux/amd64 linux/arm64 darwin/arm64; do
|
||||
goos="${target%/*}"
|
||||
goarch="${target#*/}"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||
|
||||
104
docs/cli.md
104
docs/cli.md
@@ -20,7 +20,7 @@ Top-level commands:
|
||||
- `publish <session_id>`: force-run publish.
|
||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||
- `session <subcommand>`: session helper commands.
|
||||
- `config <subcommand>`: validate, display, source-trace, or compare resolved pipeline configuration.
|
||||
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
|
||||
|
||||
Session subcommands:
|
||||
|
||||
@@ -45,6 +45,7 @@ Most session-aware commands accept:
|
||||
- `--session-id <session_id>`
|
||||
- `--previous-session-id <session_id>`
|
||||
- `--profile <name>`
|
||||
- `--secrets-dir <directory>`
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -53,10 +54,13 @@ Rules:
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `--previous-session-id` is a strict expectation: the selected session file
|
||||
must contain the same `previous_session_id`.
|
||||
- `--profile` selects a declared pipeline profile. It may be supplied once;
|
||||
an explicit empty or unknown value fails configuration resolution. When it is
|
||||
omitted, a declared `default_profile` is used. The same selection applies to
|
||||
all common-flag commands, including `regenerate-artifacts`.
|
||||
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
|
||||
may be supplied once; omitted, empty, and the exact name `default` select the
|
||||
base configuration, while an unknown named profile fails. The same selection
|
||||
applies to all common-flag commands, including `regenerate-artifacts`.
|
||||
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
|
||||
relative value uses caller CWD. Without the flag, a present `secrets/`
|
||||
directory beside the resolved root pipeline is used; an absent one is skipped.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
- notification delivery is currently limited to the configured `noop` mode; see
|
||||
the [configuration reference](./config.md#notifications).
|
||||
@@ -78,19 +82,24 @@ Commands with additional positionals keep their command-specific order:
|
||||
|
||||
## Command Reference
|
||||
|
||||
### `config validate`, `config show`, `config sources`, and `config diff`
|
||||
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
|
||||
|
||||
```bash
|
||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]
|
||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
|
||||
```
|
||||
|
||||
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
||||
a campaign is selected—the campaign-owned party. They neither discover or load
|
||||
a session nor create a workspace, manifest, run, lock, adapter, remote
|
||||
connection, or credential environment.
|
||||
a session nor create a workspace, manifest, run, lock, adapter, or remote
|
||||
connection. Like every pipeline-loading command, they bootstrap the selected or
|
||||
conventional secrets directory before resolving configuration; `config show`
|
||||
never includes bootstrap reports or secret values in its YAML output.
|
||||
Typed credential references, including `notarius.llm_api_key`, must therefore
|
||||
resolve even though these inspection commands never construct an adapter.
|
||||
|
||||
Campaign selection is optional for a pipeline without party-driven artifact
|
||||
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
||||
@@ -101,20 +110,31 @@ Session, range, force, and artifact-execution flags are not accepted.
|
||||
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
||||
effective-digest summary after successful complete validation. `config show`
|
||||
writes one deterministic, secret-free YAML document containing defaulted and
|
||||
expanded concrete configuration. It omits composition declarations, artifact
|
||||
expanded concrete configuration. It omits ConfigKit directives, artifact
|
||||
family declarations, and runtime provenance.
|
||||
|
||||
`config sources` reports the same fully validated resolution without printing
|
||||
effective values. Its header identifies the root, ordered imports, selected
|
||||
profile and overlay, selected campaign, party mode/source, and digest. The
|
||||
profile and overlay, selected campaign, party source, and digest. The
|
||||
remaining tab-separated records are sorted as `path`, `role`, and `source`.
|
||||
Roles distinguish root, import, profile, centralized default, campaign, party,
|
||||
legacy-player, and generated family ownership. A generated party member has
|
||||
and generated family ownership. A generated party member has
|
||||
one family record and one party record at the same logical path. The output
|
||||
never reads or prints secret values.
|
||||
|
||||
`config diff` resolves both supplied profile names from one parsed root source
|
||||
set and compares their fully resolved, secret-free effective mappings. It does
|
||||
`config artifacts` prints the concrete catalog used by analysis, ordered by
|
||||
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
||||
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||
lists, and control characters in fields are escaped. Family-member rows identify
|
||||
their family, character, defining family file, and party file. An empty catalog
|
||||
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
||||
party-driven families retain the campaign requirement. See
|
||||
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||
configuration layout.
|
||||
|
||||
`config diff` resolves both supplied profile names independently and compares
|
||||
their fully resolved, secret-free effective mappings. It does
|
||||
not accept `--profile`; the two positional names must be distinct, declared
|
||||
profiles. When party-driven families are present, both profiles must resolve to
|
||||
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
||||
@@ -153,17 +173,22 @@ Behavior:
|
||||
|
||||
- evaluates one inclusive contiguous range of the canonical stage order;
|
||||
- defaults an omitted `--from` to `prepare` and an omitted `--through` to
|
||||
`notify`, so omitting both retains full-pipeline behavior;
|
||||
`cleanup`, so omitting both retains full-pipeline behavior;
|
||||
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
||||
- runs `render` before `extract`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
- resolves every selected typed credential during the initial configuration
|
||||
load, including a Notarius credential when extract is disabled, outside the
|
||||
selected range, or later skipped as resumable;
|
||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||
resume check finds its durable result obsolete;
|
||||
- applies `--force` only to stages in the selected range;
|
||||
- rejects repeated `--from`, `--through`, or `--force` options, including
|
||||
`--name=value` spellings;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes session and run manifests.
|
||||
- writes the session manifest as the sole durable progress ledger. Each
|
||||
invocation also creates its run-local directory and returns an immediate
|
||||
command summary; neither creates another durable progress record.
|
||||
- reports the resolved profile (or `none`) and effective configuration digest.
|
||||
|
||||
When `--artifacts` is present, the selected range must contain `analyze` or
|
||||
@@ -187,6 +212,10 @@ artifacts and any prerequisites required by those targets; without
|
||||
never run. Common session/configuration options and repeatable artifact values
|
||||
pass through unchanged.
|
||||
|
||||
The forced extraction rebuilds the complete configured Notarius bundle,
|
||||
disables Notarius checkpoint loading, and requests chunk-plan refresh.
|
||||
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
|
||||
|
||||
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
||||
supply those options. The shared `run` parser reports them as duplicate
|
||||
singleton flags. The alias has no private execution options or behavior, and
|
||||
@@ -211,6 +240,7 @@ Valid stage names:
|
||||
- `analyze`
|
||||
- `publish`
|
||||
- `notify`
|
||||
- `cleanup`
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -228,6 +258,10 @@ Equivalent to:
|
||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
This selects only `analyze`: it does not invoke Notarius, replace extraction,
|
||||
or remove existing extract outputs. `--artifacts` narrows the forced analysis
|
||||
targets and their required analysis prerequisites.
|
||||
|
||||
### `publish`
|
||||
|
||||
```bash
|
||||
@@ -240,6 +274,10 @@ Equivalent to:
|
||||
narratio run-stage publish <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
This command selects only `publish`; it does not invoke cleanup. Use a normal
|
||||
full run to include configured cleanup, or `run-stage cleanup` to retry cleanup
|
||||
from valid publication evidence.
|
||||
|
||||
### `clean`
|
||||
|
||||
```bash
|
||||
@@ -253,8 +291,8 @@ Behavior:
|
||||
- `--all` removes all local session work and spool state;
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
|
||||
post-publish cleanup behavior.
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for manual and pipeline-stage
|
||||
deletion scope.
|
||||
|
||||
### `session plan`
|
||||
|
||||
@@ -279,7 +317,8 @@ execution; the preview does not predict output identities.
|
||||
narratio session validate <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
||||
Read-only preflight checks for config validity, required inputs, audio mode,
|
||||
previous-session requirements, publish outputs, and independent remote locks.
|
||||
|
||||
### `session status`
|
||||
|
||||
@@ -288,7 +327,12 @@ narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, status for the
|
||||
pointer-selected remote commit and its declared published outputs.
|
||||
pointer-selected remote commit and its declared published outputs. Current
|
||||
publish metadata reports `published`, `carried_locked`, `locked_unpublished`,
|
||||
`optional_missing`, and `unselected`; carried output lines include the run
|
||||
derived from their exact declared key, the key itself, and its declared
|
||||
destination. A separately displayed `dest` is the destination requested by the
|
||||
current configuration and may differ while an older declaration is locked.
|
||||
|
||||
### `session init`
|
||||
|
||||
@@ -307,6 +351,7 @@ Options:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||
- `--secrets-dir <directory>`
|
||||
- `--previous-session-id <id>`
|
||||
- `--date <YYYY-MM-DD>`
|
||||
- `--title <text>`
|
||||
@@ -346,7 +391,8 @@ narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability.
|
||||
remote published-state availability. Remote inspection uses exact commit
|
||||
declarations and displays the five current publish outcomes when recorded.
|
||||
|
||||
### `session locks`
|
||||
|
||||
@@ -358,12 +404,14 @@ narratio session locks remove <session_id> <source> [...common config flags]
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode reports the effective merge of static and remote locks;
|
||||
- add/remove mutate only remote locks;
|
||||
- static locks from pipeline config cannot be removed by CLI commands.
|
||||
- list mode validates and reports independent remote locks sorted by source;
|
||||
- add creates one lock and reports an existing lock unless `--force` is used;
|
||||
- `--force` performs one generation-matched replacement and reports a
|
||||
concurrent change for operator retry; and
|
||||
- remove is idempotent and reports when the source was not locked.
|
||||
|
||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||
and precedence.
|
||||
and publication behavior.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
|
||||
264
docs/config.md
264
docs/config.md
@@ -44,13 +44,16 @@ the durable copied session input.
|
||||
|
||||
### Read-only effective pipeline inspection
|
||||
|
||||
`narratio config validate`, `narratio config show`, and `narratio config
|
||||
sources` use the same `--config`, `--campaign`, `--campaign-file`, and
|
||||
`narratio config validate`, `narratio config show`, `narratio config sources`,
|
||||
and `narratio config artifacts` use the same `--config`, `--campaign`,
|
||||
`--campaign-file`, and
|
||||
`--profile` selection rules as pipeline commands, but do not select, discover,
|
||||
or load a session. They do not read credential values or create runtime state.
|
||||
or load a session. They resolve configured typed credential references as part
|
||||
of the ordinary ConfigKit load but expose only their environment-variable names;
|
||||
they do not create runtime state.
|
||||
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
||||
campaign selectors, resolves each named profile independently from one parsed
|
||||
root source set, and does not accept a separate `--profile` flag.
|
||||
campaign selectors, resolves each named profile with an independent ConfigKit
|
||||
load, and does not accept a separate `--profile` flag.
|
||||
|
||||
Campaign selection is optional only when the resolved pipeline has no
|
||||
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
||||
@@ -59,11 +62,12 @@ then parses the campaign-owned party and expands concrete artifacts and any
|
||||
family publish rules before validation. `config validate` prints the resulting
|
||||
root, profile, and effective digest. `config show` emits the normalized
|
||||
effective pipeline YAML, with defaults and concrete expansion included but
|
||||
composition and family declarations omitted. `config sources` prints a stable
|
||||
ConfigKit directives and family declarations omitted. This review projection is
|
||||
not promised to be a reloadable authoring document. `config sources` prints a stable
|
||||
source projection instead of effective values: root/import/profile/default
|
||||
ownership plus campaign/party and generated-family records. Canonical derived
|
||||
players trace to the party; a legacy configured players file is explicitly
|
||||
marked as a legacy player source. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
||||
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
owns command syntax and output conventions.
|
||||
|
||||
`config diff` compares normalized field values rather than YAML text or source
|
||||
@@ -77,14 +81,15 @@ difference.
|
||||
### Migrating to the maintained bundle
|
||||
|
||||
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||
as the complete copyable migration reference. Split stable pipeline settings
|
||||
into explicit additive imports, place production/testing differences in one
|
||||
selected overlay, and retain a production `default_profile`. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), remove a separate
|
||||
`players_file`, then express character work as families. Inspect the result
|
||||
with `config validate`, `config show`, and `config sources`; use `config diff`
|
||||
to review profiles before running a session. Unversioned parties and their
|
||||
`players_file` remain a clearly bounded legacy compatibility path.
|
||||
as the complete copyable migration reference. Keep production settings in the
|
||||
base, split them into explicit additive imports, and place testing differences
|
||||
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
|
||||
in the recommended one-definition files shown by that bundle. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), then express character
|
||||
work as families. Inspect the result with `config validate`, `config show`,
|
||||
`config sources`, and `config artifacts`; use `config diff default testing` to
|
||||
review the overlay before running a session. Unversioned party documents are
|
||||
rejected.
|
||||
|
||||
### Identity segments
|
||||
|
||||
@@ -97,51 +102,51 @@ remote state with an unsafe legacy identity must be migrated before use.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
||||
unknown fields or trailing documents fail load.
|
||||
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
|
||||
unknown fields, duplicate keys, and trailing documents fail load.
|
||||
- A pipeline file may explicitly import additive YAML fragments through the
|
||||
root-only `composition.imports` list. Imported files contribute fields to one
|
||||
root-only `configkit.imports` list. Imported files contribute fields to one
|
||||
logical pipeline document; they do not override fields supplied by the root
|
||||
or another import.
|
||||
- A root pipeline may declare named profiles. Exactly one profile is selected
|
||||
by an option-aware caller or by `composition.default_profile`; a caller's
|
||||
explicit selection takes precedence. Declaring profiles without either form
|
||||
of selection is an error.
|
||||
- An omitted profile, an empty profile option, and the exact name `default`
|
||||
select the base configuration. Any other name selects
|
||||
`profiles/<name>.yml` relative to the pipeline root.
|
||||
- Campaign, session, and canonical-party documents are strict single-source
|
||||
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
|
||||
directives.
|
||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Session and party inputs inspected before typed loading are limited to 8 MiB.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Defaults apply only to absent fields. Explicit empty strings, zero values,
|
||||
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
|
||||
Narratio domain validation; they are not silently replaced by defaults.
|
||||
Explicit null optional stage sections stay absent, while required sections
|
||||
fail validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||
`party_file`) and the optional `spell_catalog_file` resolve from session
|
||||
overrides when provided, otherwise from campaign defaults. An empty or
|
||||
omitted session spell-catalog value inherits the campaign value.
|
||||
- `party_file` is classified when pipeline and campaign configuration are
|
||||
combined. A versioned [canonical party](integrations/party.md) is
|
||||
campaign-owned, derives the players input internally, and forbids both a
|
||||
separate `players_file` and a session `party_file` override. An unversioned
|
||||
party remains a bounded legacy input and requires `players_file`; its normal
|
||||
campaign/session overrides continue to apply.
|
||||
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||
session. The optional `spell_catalog_file` also inherits a campaign value
|
||||
when omitted by the session.
|
||||
- `party_file` is parsed when pipeline and campaign configuration are combined.
|
||||
It must be a versioned [canonical party](integrations/party.md), is
|
||||
campaign-owned and derives the players input internally. `players_file` and
|
||||
session `party_file` are unsupported configuration keys.
|
||||
- Exactly one audio mode must be configured in session input:
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
|
||||
### Pipeline composition
|
||||
### Pipeline imports and profiles
|
||||
|
||||
Large pipeline configurations may be split into explicitly named fragments and
|
||||
may declare one overlay per selectable profile:
|
||||
may use fixed-path profile overlays:
|
||||
|
||||
```yaml
|
||||
composition:
|
||||
configkit:
|
||||
imports:
|
||||
- config/storage.yml
|
||||
- config/integrations.yaml
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: profiles/testing.yml
|
||||
- path: config/storage.yml
|
||||
- path: config/integrations.yaml
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
@@ -155,36 +160,29 @@ Imports are resolved relative to the directory containing the root pipeline
|
||||
file and are loaded in declaration order. Narratio does not scan directories or
|
||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
||||
import of the root pipeline itself are rejected. Only the root pipeline may
|
||||
contain `composition`; nested composition is rejected.
|
||||
import of the root pipeline itself are rejected. Imported files cannot declare
|
||||
another `configkit` control section.
|
||||
|
||||
Composition is additive. A map may be extended by multiple files when every
|
||||
Base merging is additive. A map may be extended by multiple files when every
|
||||
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
||||
more than once, even when the repeated values are identical. Conflict errors
|
||||
name the full field path and every source that claimed it. The assembled YAML is
|
||||
then decoded against the normal strict pipeline schema and defaults are applied
|
||||
once.
|
||||
name the full field path and every source that claimed it. Narratio additionally
|
||||
rejects an artifact or artifact-family key declared by more than one base file,
|
||||
even when those files contribute disjoint children.
|
||||
|
||||
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
|
||||
control characters. If `profiles` is present, it must contain at least one
|
||||
entry and every entry must contain only an `overlay` path. An explicit profile
|
||||
selection overrides `default_profile`; unknown and explicitly empty selections
|
||||
fail. Narratio never selects the first profile implicitly and does not read a
|
||||
profile selection from the environment.
|
||||
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
|
||||
rules. Only the selected overlay is opened; a malformed unselected overlay does
|
||||
not affect a base or different-profile load. Maps merge recursively, overlay
|
||||
scalars replace base scalars, and overlay lists replace base lists completely.
|
||||
Explicit `false`, zero, empty strings, empty containers, and null values remain
|
||||
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
|
||||
stack, import files, or declare another ConfigKit control section.
|
||||
|
||||
Every declared overlay is resolved relative to the root pipeline directory and
|
||||
must satisfy the same confined regular-YAML-file rules as an import. Narratio
|
||||
parses every declared overlay even when it is not selected, then applies only
|
||||
the selected one. Maps merge recursively, overlay scalars replace base scalars,
|
||||
and overlay lists replace base lists completely. Explicit `false`, zero, empty
|
||||
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
|
||||
kind changes are rejected. Profiles cannot inherit from or stack with other
|
||||
profiles, and overlays cannot import files or declare profiles.
|
||||
|
||||
After composition, Narratio strictly decodes the result, applies centralized
|
||||
defaults once, resolves ordinary paths, and computes a deterministic effective
|
||||
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
|
||||
entry-level defaults and normalization, read-only file-local validation, and
|
||||
ordinary path resolution before computing a deterministic effective
|
||||
configuration digest. The digest represents the normalized, secret-free
|
||||
runtime pipeline mapping; it excludes composition declarations, source
|
||||
runtime pipeline mapping; it excludes ConfigKit directives, source
|
||||
provenance, profile identity, and raw environment secret values. Equivalent
|
||||
effective mappings therefore have the same digest regardless of how fields are
|
||||
split among the root and imports.
|
||||
@@ -213,7 +211,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
```
|
||||
|
||||
@@ -228,15 +225,29 @@ inputs:
|
||||
## Secrets Handling
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
||||
environment-variable filename supplies one value; trailing CR/LF is removed.
|
||||
- Credential fields contain environment-variable names, represented internally
|
||||
as ConfigKit typed secret references (for example
|
||||
`pipeline.notarius.llm_api_key: NOTARIUS_LLM_API_KEY`). A configured reference
|
||||
must resolve to a nonempty environment value during load.
|
||||
- Every surviving reference is resolved during ordinary pipeline loading,
|
||||
regardless of whether its integration is disabled, outside the selected
|
||||
stage range, or eligible for resume. Omit the field or set it to `null` when
|
||||
that deployment should not require the credential.
|
||||
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
|
||||
flag is omitted, Narratio uses a present `secrets/` directory beside the
|
||||
resolved root `pipeline.yml`; an absent conventional directory is ignored.
|
||||
Explicit relative paths are resolved from the caller's working directory,
|
||||
and a missing or empty explicit selection is an error.
|
||||
- Each immediate regular file whose name is a valid environment-variable name
|
||||
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
|
||||
selected directory, rejects symlink entries, and limits input to 1 MiB per
|
||||
file, 8 MiB total, and 1024 entries.
|
||||
- An existing process environment value takes precedence over a credential file.
|
||||
- Credential directories and files must not be symlinks and must be regular,
|
||||
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
||||
with no group/other access (normally `0700`) and files with no group/other
|
||||
access (normally `0600`).
|
||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||
- On POSIX, operators must provision the directory with no group/other access
|
||||
(normally `0700`) and files with no group/other access (normally `0600`);
|
||||
ConfigKit validates type and bounds but does not enforce permission modes.
|
||||
- Bootstrap happens once per command, before any configuration load or adapter
|
||||
construction. Safe reports contain filenames, paths, and statuses, never values.
|
||||
|
||||
## Publish Configuration Summary
|
||||
|
||||
@@ -259,9 +270,6 @@ publish:
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: manual post-publish edits
|
||||
```
|
||||
|
||||
Rules:
|
||||
@@ -272,7 +280,8 @@ Rules:
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
- publish locks are independent remote session objects managed with
|
||||
`narratio session locks`; they are not pipeline configuration fields.
|
||||
|
||||
## Full Schema
|
||||
|
||||
@@ -280,24 +289,21 @@ Rules:
|
||||
|
||||
| Field | Type | Required | Default / Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
|
||||
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
|
||||
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
|
||||
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||
@@ -306,9 +312,6 @@ Rules:
|
||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||
| `pipeline.publish.locks[]` | list | No | empty |
|
||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
@@ -326,7 +329,7 @@ Rules:
|
||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||
| `pipeline.audita.binary` | string | No | `audita` |
|
||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
||||
| `pipeline.audita.llm_api_key` | secret reference | No | unset |
|
||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||
| `pipeline.audita.base_url` | string | No | empty |
|
||||
| `pipeline.audita.model` | string | No | empty |
|
||||
@@ -354,10 +357,13 @@ Rules:
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
|
||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
|
||||
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
|
||||
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||
| `pipeline.render.enabled` | bool | No | `true` |
|
||||
@@ -374,6 +380,48 @@ Rules:
|
||||
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
||||
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||
|
||||
### Notarius Credential
|
||||
|
||||
`pipeline.notarius.llm_api_key` optionally names the one environment variable
|
||||
to forward to the Notarius child process:
|
||||
|
||||
```yaml
|
||||
notarius:
|
||||
llm_api_key: OPENROUTER_API_KEY
|
||||
```
|
||||
|
||||
The field contains the variable name, not a credential value. That name must
|
||||
match `api_key_env` in the Notarius/PromptKit profile selected by the deployed
|
||||
Notarius pipeline; Narratio does not infer or remap provider names. Omission or
|
||||
`null` forwards nothing. A non-null selected reference must resolve at pipeline
|
||||
load time through the process environment or existing secrets-directory
|
||||
bootstrap, even when Notarius is disabled, extract is outside the requested
|
||||
range, or a completed extraction can be reused.
|
||||
|
||||
The Notarius child receives only Narratio's minimal execution environment plus
|
||||
this exact selected name and loaded value. Other credentials are not forwarded.
|
||||
Changing the value, name, presence, or absence does not invalidate extract's
|
||||
semantic resume evidence. The public name remains part of safe effective
|
||||
inspection, so renaming it may appear in `config diff` and change the effective
|
||||
configuration digest; rotating only its value does neither.
|
||||
|
||||
See the [Notarius integration contract](./integrations/notarius.md) for the
|
||||
delivery and redaction boundary.
|
||||
|
||||
### Notarius Cache Controls
|
||||
|
||||
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
|
||||
extraction passes `--resume`; the deployed Notarius configuration must already
|
||||
enable checkpoint recording and should select an explicit persistent checkpoint
|
||||
root. Narratio does not inspect or generate that configuration.
|
||||
|
||||
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
|
||||
configured mode in control. A nonempty value must be `auto`, `bypass`, or
|
||||
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
|
||||
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
|
||||
Neither field changes Narratio's semantic extraction fingerprint, so changing
|
||||
them alone does not rerun an otherwise complete durable extraction.
|
||||
|
||||
### Notarius Reference Bindings
|
||||
|
||||
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
||||
@@ -403,8 +451,8 @@ It validates only selector structure and the prepared source vocabulary;
|
||||
Notarius owns target-slot declarations and media compatibility.
|
||||
|
||||
Before extraction, Narratio resolves every binding from the current prepared
|
||||
session manifest and streams it into a verified invocation-local snapshot whose
|
||||
absolute path is passed to Notarius. Missing, unsafe, empty,
|
||||
session manifest and streams it into a verified, content-addressed session
|
||||
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
|
||||
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
||||
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
||||
part of extraction fingerprint and resume identity. See the
|
||||
@@ -439,7 +487,7 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | bool | No | `false` if omitted |
|
||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; configured graph must be acyclic |
|
||||
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
|
||||
| `render_debug` | bool | No | per-artifact override |
|
||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||
| `profile_id` | string | No | empty |
|
||||
@@ -453,8 +501,8 @@ Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium re
|
||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||
those targets over `depends_on`: a current prerequisite is reused, while a
|
||||
stale, missing, failed, or legacy prerequisite is rebuilt before its dependent.
|
||||
those targets over the effective dependency graph: a current prerequisite is reused, while a
|
||||
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||
may require rebuilding must therefore have valid executable fields. This
|
||||
override affects analyze planning only; publish uses the list only to filter
|
||||
@@ -471,12 +519,22 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||
does not provide adapter-specific input passthrough fields.
|
||||
|
||||
Each `narratio.artifact.<name>` input source automatically contributes an
|
||||
effective dependency on that configured artifact. Narratio unions these inferred
|
||||
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
|
||||
result during final configuration resolution. Authors use `depends_on` only for
|
||||
ordering constraints that are not already expressed by an artifact input.
|
||||
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
to inspect keys, definition sources, outputs, and effective dependencies without
|
||||
loading a session or creating runtime state.
|
||||
|
||||
### Scriptorium Artifact Families
|
||||
|
||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||
for every canonical campaign character. Configuration resolution expands each
|
||||
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
|
||||
planning or Scriptorium invocation. A legacy party cannot be used for a family.
|
||||
planning or Scriptorium invocation. Party parsing rejects unsupported document
|
||||
schemas before family expansion.
|
||||
|
||||
For each `pipeline.scriptorium.artifact_families.<name>`:
|
||||
|
||||
@@ -501,9 +559,10 @@ Their resolved values are strings. A member variable may not reuse a static
|
||||
other Scriptorium artifact.
|
||||
|
||||
Within a family only, an input source may use
|
||||
`narratio.member_artifact.<family>`. The referenced family must be named in
|
||||
that family's `member_dependencies`; resolution rewrites the source to the
|
||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source.
|
||||
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
|
||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
|
||||
infers the concrete member dependency. Use `member_dependencies` only for
|
||||
additional same-character ordering constraints that are not already inputs.
|
||||
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
||||
or Scriptorium.
|
||||
|
||||
@@ -524,7 +583,6 @@ integration.
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||
| `inputs.players_file` | string | Conditional | required only with an unversioned legacy `party_file`; forbidden for a canonical party |
|
||||
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
|
||||
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
|
||||
|
||||
@@ -540,8 +598,6 @@ integration.
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.players_file` | string | No | legacy-party override; forbidden for a canonical party |
|
||||
| `inputs.party_file` | string | No | legacy-party override; forbidden for a canonical campaign party |
|
||||
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
|
||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||
|
||||
@@ -10,6 +10,11 @@ polished transcripts and generated artifacts. Start with the
|
||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||
|
||||
## Go Prerequisite
|
||||
|
||||
Maintained development and verification require Go 1.26. Use the official Go
|
||||
1.26 toolchain before running repository commands.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
@@ -53,7 +58,7 @@ in the [Release Procedure](release.md).
|
||||
|
||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||
for macOS and Windows, but do not provide native macOS or Windows execution.
|
||||
Those cross-builds establish compilation only, not platform-equivalent runtime
|
||||
evidence. Add native checks only when official runner labels and successful
|
||||
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
|
||||
target. The macOS cross-build establishes compilation only, not native runtime
|
||||
evidence. Add a native check only when an official runner label and successful
|
||||
native-run evidence are available.
|
||||
|
||||
@@ -41,7 +41,6 @@ Construction fails for invalid static config values, including:
|
||||
|
||||
Run fails for:
|
||||
- missing required request paths;
|
||||
- missing required credential env var when configured (`llm_api_key_env`);
|
||||
- subprocess execution failure;
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
@@ -53,6 +52,9 @@ Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
## Deterministic Behavior
|
||||
- CLI args are built from runner config + request in a fixed order.
|
||||
- A configured typed `llm_api_key` reference is resolved during pipeline load;
|
||||
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
|
||||
override and is never re-read from the parent environment.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
|
||||
@@ -7,14 +7,14 @@ lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||
|
||||
Canonical Notarius v0.6.0 references:
|
||||
Canonical Notarius v0.7.0 references:
|
||||
|
||||
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/cli.md)
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/json-output.md)
|
||||
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
||||
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
|
||||
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
||||
|
||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||
@@ -25,19 +25,26 @@ duplicate the complete schemas.
|
||||
|
||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||
configuration path, input path, output directory, and working directory to
|
||||
absolute paths. Narratio requires the Notarius v0.6.0 CLI contract when
|
||||
references are configured and invokes each binding as a separate argument
|
||||
before `--json`:
|
||||
absolute paths. Configured references require support for the `--reference`
|
||||
contract introduced in Notarius v0.6.0. Cache controls require support for the
|
||||
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
|
||||
cache integration baseline, not an unconditional minimum version; Narratio
|
||||
does not impose a runtime version gate. Each reference binding is a separate
|
||||
argument before `--json`:
|
||||
|
||||
```text
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--reference <selector>=<verified_snapshot_path>]... --json
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... --json
|
||||
```
|
||||
|
||||
Reference paths are absolute invocation-local snapshots streamed from the
|
||||
manifest-verified canonical files prepared inside the current Narratio session
|
||||
workspace. Narratio verifies snapshot checksum and size before and after the
|
||||
subprocess, and passes only configured bindings, ordered lexically by normalized
|
||||
selector, as direct argument-vector entries without shell interpretation. A CLI
|
||||
Reference paths are absolute stable snapshots streamed from manifest-verified
|
||||
canonical files prepared inside the current Narratio session workspace. They
|
||||
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
|
||||
full lowercase content digest. Narratio verifies snapshot checksum and size
|
||||
before and after the subprocess, reconstructs missing or corrupt regular files
|
||||
from a still-valid prepared source, and passes only configured bindings,
|
||||
ordered lexically by normalized selector, as direct argument-vector entries
|
||||
without shell interpretation. Symlinks, directories, other non-regular
|
||||
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
|
||||
binding takes precedence over a matching external path in Notarius
|
||||
configuration. Narratio never emits `--without-reference`.
|
||||
|
||||
@@ -75,9 +82,41 @@ It does not pass a Narratio session ID or run `notarius config validate`
|
||||
automatically; the configured working directory and Narratio's minimal child
|
||||
environment apply to the subprocess.
|
||||
|
||||
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
|
||||
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
|
||||
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
|
||||
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
|
||||
Narratio neither reads nor synthesizes its configuration. Use explicit
|
||||
persistent roots in Notarius when reuse must span subprocess invocations.
|
||||
Changing a reference path/digest or other upstream identity input correctly
|
||||
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
|
||||
replacement for the active source; it does not delete unrelated plans or
|
||||
checkpoint state.
|
||||
|
||||
When `pipeline.notarius.llm_api_key` is absent or null, Narratio adds no
|
||||
credential to that environment. When configured, the field names an environment
|
||||
variable whose already-loaded nonempty value is forwarded under that exact
|
||||
name. Operators must use the same name as the `api_key_env` in the selected
|
||||
Notarius/PromptKit profile. Narratio does not inspect the Notarius configuration
|
||||
to infer a provider or destination name, and it does not forward unrelated
|
||||
credentials from its parent environment or secrets directory.
|
||||
|
||||
The selected name is registered with Narratio's shared streaming redactor.
|
||||
Credential bytes written by the child to captured stdout, captured stderr, or
|
||||
stderr-derived failure diagnostics are replaced with `<redacted>`, including
|
||||
custom names not known to the built-in sensitive-name list. Narratio does not
|
||||
serialize the credential into arguments, stage metadata, manifests, or
|
||||
generated configuration.
|
||||
|
||||
This captured-stream protection is not a filesystem sandbox. Notarius remains
|
||||
responsible for keeping credentials out of files it writes directly, including
|
||||
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
|
||||
arbitrary downstream files and cannot promise redaction of transformed or
|
||||
encoded credentials.
|
||||
|
||||
## Accepted Result
|
||||
|
||||
Narratio's supported invocation baseline is Notarius v0.6.0. The accepted
|
||||
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
|
||||
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
||||
receipt or ten-lane output contract. The receipt
|
||||
must identify the configured pipeline, and its `index_file` must be exactly
|
||||
@@ -102,6 +141,8 @@ version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||
incompatible required lanes fail extraction even if Notarius exited zero. A
|
||||
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
||||
fails extraction.
|
||||
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
|
||||
preserves that valid summary rather than treating it as a missing field.
|
||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||
selectable Narratio sources.
|
||||
|
||||
@@ -130,3 +171,18 @@ Configuration fields and defaults are in [Configuration](../config.md).
|
||||
Operator paths, rerun procedures, and bundle retention are in
|
||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||
for failure recovery.
|
||||
|
||||
## Opt-in Integration Test
|
||||
|
||||
The production-boundary cache test requires an absolute path to a locally
|
||||
built Notarius v0.7.0-compatible executable. It uses only a local
|
||||
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
|
||||
|
||||
```sh
|
||||
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
|
||||
GOWORK=off go test ./internal/stage \
|
||||
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
|
||||
```
|
||||
|
||||
When the variable is unset, the normal test suite skips this test and never
|
||||
searches `PATH`, a sibling checkout, the network, or a real provider.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Canonical Party Input
|
||||
|
||||
`party.yml` is a campaign-owned roster input. Narratio recognizes the
|
||||
versioned `narratio.party.v1` document below when it resolves a pipeline,
|
||||
campaign, and session together.
|
||||
`party.yml` is a campaign-owned roster input. It must be a single, versioned
|
||||
`narratio.party.v1` document when Narratio resolves a pipeline, campaign, and
|
||||
session together.
|
||||
|
||||
```yaml
|
||||
schema_version: narratio.party.v1
|
||||
@@ -34,19 +34,17 @@ non-empty, trimmed display strings without control characters. Character names
|
||||
and aliases must be unique across the full roster under Unicode-aware
|
||||
case-insensitive comparison; player names may repeat.
|
||||
|
||||
The document has exactly one YAML document and accepts no unknown fields. A
|
||||
wrong or malformed `schema_version` is an error.
|
||||
The document has exactly one YAML document and accepts no unknown fields.
|
||||
`schema_version` is required: a missing, wrong, or malformed value is an error.
|
||||
|
||||
## Legacy migration boundary
|
||||
## Migration
|
||||
|
||||
An unversioned party input remains supported only as opaque legacy reference
|
||||
material while campaigns migrate. It requires a separate `players_file` and
|
||||
retains the existing session override behavior. It cannot be mixed with a
|
||||
canonical party: canonical campaigns must omit `players_file`, and sessions
|
||||
must not override their party or players inputs.
|
||||
Unversioned party inputs are rejected. Replace each legacy party document with
|
||||
the versioned structure above, move each roster entry under `characters`, and
|
||||
remove separate player and session-party settings: those keys are unsupported.
|
||||
|
||||
Use the canonical document for new campaigns. The configuration rules and
|
||||
source-relative path behavior are defined in the [Configuration Reference](../config.md).
|
||||
The configuration rules and source-relative path behavior are defined in the
|
||||
[Configuration Reference](../config.md).
|
||||
|
||||
## Derived players document
|
||||
|
||||
|
||||
@@ -35,8 +35,9 @@ Adapters do not own:
|
||||
|
||||
## Default Wiring
|
||||
|
||||
`internal/app/runner.go` initializes default adapters when not injected and
|
||||
only when the selected execution plan needs them:
|
||||
The application composes default adapters per invocation when they were not
|
||||
injected through the stage environment, and only when the selected execution
|
||||
plan needs them:
|
||||
|
||||
- WhisperX HTTP client for `transcribe`.
|
||||
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
|
||||
@@ -49,8 +50,10 @@ only when the selected execution plan needs them:
|
||||
Remote publish locks are loaded only for a selected, enabled publish that
|
||||
uploads a run. Shared session lifecycle setup still applies to every selected
|
||||
range, but an unselected integration is neither initialized nor validated by
|
||||
runner composition. Each selected stage retains its own fail-fast configuration
|
||||
and input validation.
|
||||
application-owned composition. Constructor, clock, randomness, remote-lock,
|
||||
and post-publish deletion boundaries are scoped to the invocation; stages still
|
||||
receive their collaborators through the stage environment. Each selected stage
|
||||
retains its own fail-fast configuration and input validation.
|
||||
|
||||
`session plan` is outside production adapter composition. It performs
|
||||
resume validation and models selected transitions against cloned manifest
|
||||
@@ -62,31 +65,44 @@ Notarius is composed only when extraction is enabled; the extract stage owns
|
||||
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
||||
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
||||
owner of serializing them as repeated `--reference` arguments before `--json`.
|
||||
It also serializes an effective resume request as `--resume` and a nonempty
|
||||
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
|
||||
adapter, resolves force precedence before constructing the request.
|
||||
Its request may also carry one already-resolved opaque ConfigKit secret
|
||||
reference. Only the Notarius adapter reads that value, rejects an unresolved or
|
||||
empty programmatic reference, and maps the exact configured name/value into the
|
||||
shared subprocess override and sensitive-name fields. It never rereads the
|
||||
parent environment. A nil reference preserves the previous invocation.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||
configured filesystem secrets before adapter initialization.
|
||||
Application-owned object-store construction loads configured filesystem
|
||||
secrets before invoking the invocation's storage factory. Production commands
|
||||
use the concrete storage adapter constructor; focused tests can supply an
|
||||
in-memory backend without changing package state.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
- Shared subprocess execution starts an owned process group on Linux/macOS or a
|
||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
||||
owned tree before returning. After a natural leader exit, Unix checks for
|
||||
remaining group members and uses bounded graceful then forceful termination;
|
||||
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
||||
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||
an owned process group on Linux and macOS. Every terminal path disposes of
|
||||
that owned tree before returning. After a natural leader exit, the runner checks
|
||||
for remaining group members and uses bounded graceful then forceful termination.
|
||||
Cancellation, deadlines, and
|
||||
diagnostic limits use the same terminal disposal path without losing their
|
||||
original result classification. Child environments contain only the execution
|
||||
baseline and adapter-specified values; configured credentials are explicit
|
||||
sensitive values. Stdout and stderr are redacted while streaming into separate
|
||||
baseline and adapter-specified values; configured credentials are resolved
|
||||
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
|
||||
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
|
||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||
departed leader's descendant. Unsupported platforms reject owned command
|
||||
execution.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||
- Composition: `internal/app/runner_composition.go`,
|
||||
`internal/app/runner_dependencies.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
|
||||
@@ -20,8 +20,8 @@ The internal registry recognizes these stable built-in source IDs:
|
||||
- `narratio.transcript.final_trimmed_markdown`
|
||||
- `narratio.bounds.session`
|
||||
|
||||
Registry entries bind each ID to its producer, output kind, canonical fallback,
|
||||
and content validator. The focused stage documents own their input/output flow;
|
||||
Registry entries bind each ID to its producer, output kind, and content
|
||||
validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
@@ -58,10 +58,12 @@ Configured definitions are always registered. Without an explicit selection,
|
||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||
exact named configured definitions become the effective set for that invocation,
|
||||
regardless of their `enabled` value. The effective-set resolver itself does not
|
||||
expand dependencies; the analyze work planner closes those targets over their
|
||||
configured prerequisite graph. Availability is separate from executability.
|
||||
Configuration may normalize a family selection into its concrete generated
|
||||
members before this resolver runs. The effective set retains optional family
|
||||
expand dependencies. Configuration has already materialized the sorted union of
|
||||
explicit ordering edges and configured-artifact input edges, and the analyze
|
||||
work planner closes targets over that one concrete prerequisite graph.
|
||||
Availability is separate from executability. Configuration may normalize a
|
||||
family selection into its concrete generated members before this resolver runs.
|
||||
The effective set retains optional family
|
||||
and character origin metadata, but its keys, catalog sources, and runtime
|
||||
lookups remain concrete configured-artifact identities.
|
||||
Family publish policies are likewise expanded into ordinary configured-source
|
||||
@@ -69,8 +71,9 @@ publish rules during configuration resolution.
|
||||
Configured outputs, including non-executable prerequisites, become available
|
||||
only when the versioned analyze state identifies a current result whose source,
|
||||
contract, canonical configured path, size, and checksum match a confined
|
||||
no-follow regular file. An incidental canonical file and a legacy aggregate
|
||||
analyze output are unavailable.
|
||||
no-follow regular file. An incidental canonical file is unavailable, and a
|
||||
manifest with an unsupported aggregate-only representation is rejected before
|
||||
catalog construction.
|
||||
Extraction entries are registered from configuration and become available only
|
||||
after compatible extraction evidence is hydrated.
|
||||
|
||||
@@ -91,8 +94,13 @@ Current provenance values:
|
||||
|
||||
Built-ins:
|
||||
|
||||
1. manifest producer outputs (when present)
|
||||
2. canonical session-path fallback
|
||||
- resolve only from the current manifest's succeeded registered producer;
|
||||
- require exactly one output with the registered source ID and output kind, a
|
||||
safe session-rooted path, producer run ID, canonical checksum, and valid
|
||||
content; and
|
||||
- verify the registered checksum against the durable bytes. Incidental files,
|
||||
canonical filenames, duplicate outputs, failed/stale producer records, and
|
||||
malformed evidence are unavailable.
|
||||
|
||||
Configured sources (`narratio.artifact.*`):
|
||||
|
||||
@@ -103,7 +111,7 @@ Configured sources (`narratio.artifact.*`):
|
||||
record for the exact configured key and source ID, a complete contract, the
|
||||
configured canonical relative path, positive stored size, and stored
|
||||
checksum matching bytes read from a confined no-follow regular file; and
|
||||
- treat non-current statuses, legacy or malformed records, removed keys,
|
||||
- treat non-current statuses, malformed records, removed keys,
|
||||
unsafe or missing files, and size/checksum mismatches as unavailable without
|
||||
rewriting manifest state. Catalog construction iterates current
|
||||
configuration, so removed or renamed records are not advertised.
|
||||
@@ -135,9 +143,12 @@ Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
- prefer manifest-backed previous-input paths;
|
||||
- fallback to existing previous-cache filesystem paths.
|
||||
- resolve only from one matching `previous_artifact` input record in the
|
||||
current manifest, including its exact source ID and checksum;
|
||||
- are hydrated only from the unique `published_output` declaration for their
|
||||
canonical source in the selected remote commit; and
|
||||
- never infer a path from a previous manifest, current configuration, basename,
|
||||
or existing cache file.
|
||||
|
||||
Source absence is evaluated by the consuming artifact input. An optional input
|
||||
is omitted from that invocation; a required input fails resolution. This is
|
||||
@@ -165,25 +176,34 @@ Validation by content type:
|
||||
Artifacts package owns shared remote current-state loading mechanics used by
|
||||
restore, status and validation checks, and previous-cache planning.
|
||||
|
||||
The selected session manifest is accepted only with
|
||||
`schema_version: narratio.session-manifest.v1` and complete stage identity.
|
||||
An unsupported local or downloaded manifest must be replaced by regeneration,
|
||||
verified restore, or republication; artifact resolution never fills in missing
|
||||
authority markers.
|
||||
|
||||
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||
complete restore authority. Callers receive its declared object identities and
|
||||
must not supplement them by listing mutable session prefixes. The legacy reader
|
||||
is intentionally separate and remains migration-only support.
|
||||
must not supplement them by listing mutable session prefixes.
|
||||
|
||||
A `published_output` declaration must use a canonical publish-compatible source
|
||||
ID and an exact `runs/{run_id}/published/{destination}` key in the same session.
|
||||
Only this declaration type may refer to an earlier run; other declared objects
|
||||
remain confined to the selected commit's run.
|
||||
|
||||
The reader opens each small control object directly and enforces owner-specific
|
||||
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
|
||||
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
|
||||
manifest limit to `current/manifest.json`. These are exposed as
|
||||
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
|
||||
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
|
||||
`MaxLegacyCurrentManifestBytes`.
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. These
|
||||
are exposed as `MaxCurrentCommitPointerBytes`,
|
||||
`MaxRemoteCommitManifestBytes`, and `MaxRemoteSessionManifestBytes`.
|
||||
|
||||
Each read uses the generation and size metadata returned with its opened body.
|
||||
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||
checksum, generation, and identity checks. No current-state control object is
|
||||
downloaded through a temporary file.
|
||||
downloaded through a temporary file. Artifact and previous-session downloads
|
||||
validate their complete sibling temporary file against available committed
|
||||
identity before replacing a canonical local path.
|
||||
|
||||
Core helpers:
|
||||
|
||||
@@ -191,10 +211,9 @@ Core helpers:
|
||||
- `ValidateCurrentStateIdentity`
|
||||
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||
|
||||
Typed missing-state errors:
|
||||
Typed missing-state error:
|
||||
|
||||
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
||||
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
||||
- `CurrentCommitPointerMissingError` (`ErrCurrentCommitPointerMissing`)
|
||||
|
||||
Identity validation supports caller-provided expectations:
|
||||
|
||||
@@ -208,6 +227,21 @@ Caller policy is intentionally outside artifacts helpers:
|
||||
- some callers downgrade missing state to status/findings;
|
||||
- some callers skip optional behavior when state is missing.
|
||||
|
||||
## Remote Lock Objects
|
||||
|
||||
The artifacts package defines the independent remote-lock payload and identity
|
||||
contract. Each canonical publish source maps to one object below the validated
|
||||
session `locks/` prefix; its filename is the lowercase SHA-256 digest of the
|
||||
exact source followed by `.lock`.
|
||||
|
||||
Lock payloads are strict versioned JSON containing the canonical source, an
|
||||
optional reason, and a UTC creation time. Reads are bounded to
|
||||
`MaxRemoteLockObjectBytes` (64 KiB), reject unknown fields and trailing values,
|
||||
and verify that the payload source matches the key digest. Exact lookup reports
|
||||
presence explicitly, prefix listing validates every object and sorts by source,
|
||||
creation uses an absence precondition, forced replacement uses one observed
|
||||
generation, and removal is idempotent.
|
||||
|
||||
## Key Path Helpers
|
||||
|
||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||
@@ -220,8 +254,13 @@ Caller policy is intentionally outside artifacts helpers:
|
||||
New publication creates run-scoped immutable objects, including
|
||||
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||
commit and declared object generations/checksums. Legacy current-pair loading
|
||||
is confined to `current_state_legacy.go` for migration only.
|
||||
commit and declared object generations/checksums.
|
||||
|
||||
A `published_output` declaration may identify its commit's run or an earlier
|
||||
immutable run in the same campaign/session namespace. The key parser validates
|
||||
that namespace, rejects mutable and malformed keys, and derives the originating
|
||||
run from the exact declaration. All other commit artifacts remain confined to
|
||||
the commit's own run prefix.
|
||||
|
||||
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||
segments at configuration and artifact boundaries before they can be used in a
|
||||
@@ -252,7 +291,6 @@ physical layout.
|
||||
`internal/artifacts/prepared_input.go`
|
||||
- Current state: `internal/artifacts/current_state.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
|
||||
@@ -11,6 +11,11 @@ physical restore scope belong in
|
||||
Restore separates remote authority, local conflict policy, and filesystem
|
||||
mutation so each remains testable independently.
|
||||
|
||||
The public command supplies production defaults to one private,
|
||||
invocation-scoped dependency value. Discovery, planning, and execution use that
|
||||
same value, so focused sequencing tests can replace one boundary without
|
||||
mutable package hooks.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Discovery delegates current-state pointer and manifest loading to
|
||||
@@ -34,18 +39,25 @@ Planner behavior:
|
||||
|
||||
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||
each action carries that artifact's immutable key, checksum, size, and
|
||||
generation. Coherent legacy state remains on the isolated compatibility path;
|
||||
generation;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted by local relative path and then remote key;
|
||||
- planning opens the selected session root and inspects every target by its
|
||||
normalized session-relative path. Missing ancestors produce a missing target;
|
||||
an ancestor or leaf symlink is a non-regular conflict and is never opened for
|
||||
hashing;
|
||||
- force converts differing eligible regular files from conflicts to downloads;
|
||||
directories and other non-regular targets remain conflicts.
|
||||
directories, symlinks, and other non-regular targets remain conflicts.
|
||||
|
||||
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||
|
||||
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||
prepare, status, and validation. A committed source is selected only by its
|
||||
exact source identity; legacy fallback remains isolated and rejects ambiguity.
|
||||
prepare, status, and validation. Each requirement selects the unique
|
||||
`published_output` for its canonical source in the pointer-selected commit,
|
||||
then strips only that run's immutable `published/` prefix for its safe local
|
||||
cache path. It never probes candidate objects or derives paths from manifests,
|
||||
configuration, or basenames.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
@@ -53,10 +65,14 @@ Execution order and safety:
|
||||
|
||||
- non-manifest downloads happen before manifest install;
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files plus atomic rename;
|
||||
- downloads use sibling temp files, validate the complete object, then use an
|
||||
atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- each committed object is verified against its declared checksum, size, and
|
||||
generation before installation;
|
||||
- installation reopens the selected session root, creates only validated
|
||||
missing directory components with workspace modes, and rejects symlinked or
|
||||
non-regular ancestors and leaves before replacement;
|
||||
- a committed manifest already verified during discovery is retained for the
|
||||
matching restore action and revalidated before installation, avoiding a
|
||||
second body transfer;
|
||||
@@ -67,15 +83,17 @@ Execution order and safety:
|
||||
- restored manifest local references are rebased beneath the selected local
|
||||
session root. Unsafe relative references and producer-machine absolute paths
|
||||
outside the manifest's producer session root are rejected; producer-local
|
||||
spool/cache and cleanup locations are not restored as authority.
|
||||
spool paths and run-local diagnostic references are not restored as
|
||||
authority. Cleanup remains ordinary restored stage state and is not reset or
|
||||
synthesized by restore.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- reuses cached audio only when its no-follow regular file, content digest, and
|
||||
- reuses cached audio only when its opened regular file, content digest, and
|
||||
identity sidecar all match the selected remote object version; otherwise it
|
||||
refreshes through the durable download path.
|
||||
refreshes through the validated download path.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
@@ -92,9 +110,7 @@ Audio restore path:
|
||||
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||
different run into its plan;
|
||||
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||
establish new-protocol remote commitment; coherent legacy
|
||||
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
|
||||
support;
|
||||
establish remote commitment; when absent, republish the session;
|
||||
- restore does not execute pipeline stages.
|
||||
|
||||
## Implementation And Tests
|
||||
@@ -107,4 +123,5 @@ Audio restore path:
|
||||
- Tests: `internal/app/restore_discovery_test.go`,
|
||||
`internal/app/restore_plan_test.go`,
|
||||
`internal/app/restore_execution_test.go`,
|
||||
`internal/app/restore_recovery_test.go`,
|
||||
`internal/app/restore_workflow_test.go`
|
||||
|
||||
@@ -8,29 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
|
||||
|
||||
`LoadPipeline` assembles and validates a pipeline in this order:
|
||||
|
||||
1. Parse the root YAML into a presence-aware composition tree. The tree retains
|
||||
source names, full field paths, node kinds, declaration order, and explicit
|
||||
zero, false, empty-map, and empty-list values.
|
||||
2. Remove the root-only `composition` envelope and validate its explicit
|
||||
`imports`, `default_profile`, and named `profiles` declarations. A load
|
||||
option retains the difference between omitted and explicitly empty profile
|
||||
selection.
|
||||
3. Open each import relative to the root pipeline directory through the
|
||||
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
|
||||
extension and cannot traverse, use symlinks, repeat a file, import the root,
|
||||
or contain another composition envelope.
|
||||
4. Resolve and structurally parse every declared profile overlay through the
|
||||
same confined regular-file boundary. Missing or malformed unselected
|
||||
overlays fail the load. Overlays cannot contain a composition envelope.
|
||||
5. Additively merge the root body and imports. Distinct map leaves compose;
|
||||
repeated scalar or list paths and node-kind disagreements are conflicts.
|
||||
6. Select exactly one declared profile from an explicit option or the default,
|
||||
then recursively merge its overlay. Overlay leaves replace base leaves,
|
||||
lists are atomic replacements, and null or kind changes fail.
|
||||
7. Emit deterministic canonical YAML and strictly decode it into
|
||||
`PipelineConfig`.
|
||||
8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from
|
||||
the root pipeline file, and digest the normalized effective mapping.
|
||||
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
|
||||
enabled imports, and the caller's profile selection. Omitted, empty, and
|
||||
exact `default` selections use the base; named profiles use the fixed
|
||||
`profiles/<name>.yml` convention.
|
||||
2. ConfigKit parses the root and declared imports, additively merges base
|
||||
sources, applies only the selected overlay, and strictly decodes the result.
|
||||
3. The definition resolver applies recorded entry-level and inherited defaults,
|
||||
runs deterministic domain normalization, records derived publish destinations,
|
||||
and resolves ordinary paths relative to the root pipeline file.
|
||||
4. The definition validator performs read-only checks that require only the
|
||||
pipeline file. Checks requiring canonical-party family expansion remain at
|
||||
the existing explicit pipeline/campaign resolution boundary.
|
||||
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
|
||||
declared by multiple base files, adapts source ownership records, attaches
|
||||
runtime-only state, and computes the normalized effective digest.
|
||||
|
||||
This ordering preserves monolithic configuration behavior. Moving a field to
|
||||
an imported fragment changes its source ownership, not its path base, default,
|
||||
@@ -52,109 +44,152 @@ download retain that exact pipeline object and its private provenance. Removing
|
||||
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||
pipeline or campaign context.
|
||||
|
||||
Campaign and session files use independent ConfigKit definitions with imports
|
||||
and profiles disabled. Campaign files normally load directly from their source;
|
||||
session files are first read through the 8 MiB bounded-read boundary because the
|
||||
application must reject unresolved template placeholders before passing an exact
|
||||
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
|
||||
the same typed loader and retain the existing post-load identity expectations,
|
||||
stable-input precedence, and source-relative path semantics.
|
||||
|
||||
The application also has a separate read-only inspection resolver for `config
|
||||
validate`, `config show`, and `config sources`. It uses the same production root/profile and
|
||||
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
|
||||
campaign selection functions, but never routes through session discovery,
|
||||
remote-session download, secret loading, adapter composition, workspace
|
||||
remote-session download, adapter composition, workspace
|
||||
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
||||
family declarations must resolve its selected campaign before ordinary pipeline
|
||||
validation, which expands its canonical-party members and generated publish
|
||||
rules. A pipeline without those declarations may be validated by itself.
|
||||
Secret-directory bootstrap remains invocation-scoped and happens before the
|
||||
ordinary ConfigKit load because explicit typed references must resolve even for
|
||||
inspection.
|
||||
|
||||
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
||||
show`. It serializes the typed, defaulted effective mapping through the
|
||||
deterministic composition renderer, then removes resolution-only artifact
|
||||
family declarations. The result contains no composition envelope or private
|
||||
provenance fields and has one trailing newline; commands do not marshal runtime
|
||||
objects directly.
|
||||
show`. It serializes one explicit typed safe projection with the existing YAML
|
||||
encoder. The projection has no slots for resolution-only artifact family
|
||||
declarations or private provenance. The result has one trailing newline;
|
||||
commands do not marshal runtime objects directly.
|
||||
|
||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||
the complete logical field paths retained during composition and classifies
|
||||
the complete logical field paths projected from the schema and classifies
|
||||
each contributor as root, import, profile, or centralized default. The
|
||||
projection replaces generated concrete member paths with paired family and
|
||||
canonical-party records, and does the same for generated publish rules.
|
||||
Campaign records identify campaign-owned fields and party inputs; canonical
|
||||
derived players point to the party source, while legacy players retain a
|
||||
dedicated legacy-player role. The application command only joins these sorted
|
||||
derived players point to the party source. The application command only joins these sorted
|
||||
records with selection metadata and never reparses configuration files.
|
||||
|
||||
`config diff` uses a paired profile loader that parses the root, imports, and
|
||||
declared overlays once, then clones the additive base before independently
|
||||
selecting, decoding, defaulting, and finalizing each profile. When campaign
|
||||
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||
family/character/party provenance for generated members. It does not inspect
|
||||
files named after artifacts or infer another runtime graph.
|
||||
|
||||
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||
loads. When campaign
|
||||
resolution is needed, the command loads one selected campaign and party and
|
||||
expands both effective pipelines from that same party value. The configuration
|
||||
owner projects each normalized effective mapping into sorted logical paths;
|
||||
owner projects each normalized effective mapping directly from that same typed
|
||||
projection into sorted logical paths using JSON number preservation;
|
||||
mapping leaves are compared individually while sequence values remain atomic.
|
||||
Values are compact deterministic JSON representations for command output, not
|
||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||
with no projected difference is treated as an internal consistency error.
|
||||
|
||||
Campaign context construction also reads and classifies the campaign-owned
|
||||
party source through `ParseParty`. A canonical party retains its raw bytes and
|
||||
normalized roster in runtime-only `ResolvedParty` provenance, while a legacy
|
||||
party remains opaque. Canonical resolution creates a virtual
|
||||
`derived_from_party` players input and rejects competing campaign or session
|
||||
players files and session party overrides. The compact legacy compatibility
|
||||
path resolves the effective campaign/session party and players files together.
|
||||
Campaign context construction reads the campaign-owned party source through
|
||||
`ParseParty`. The required canonical party retains its raw bytes and normalized
|
||||
roster in runtime-only `ResolvedParty` provenance. Canonical resolution creates
|
||||
a deterministic `derived_from_party` players projection. The party is the only
|
||||
configured roster authority; player and session-party fields are unsupported.
|
||||
|
||||
## Canonical Party Domain
|
||||
|
||||
`ParseParty` is the package-owned boundary for classifying a party source.
|
||||
When a top-level `schema_version` is present, it strictly validates the
|
||||
`narratio.party.v1` contract into ordered character domain values. The
|
||||
canonical value retains a separate exact byte copy of its source so consumers
|
||||
can materialize the authored party document without reserializing it. Its
|
||||
`PlayersYAML` method deterministically derives the versioned players-only
|
||||
projection.
|
||||
`ParseParty` is the package-owned boundary for parsing a party source. Its
|
||||
single-source ConfigKit definition disables imports and profiles, requires
|
||||
`schema_version` through `VersionRule`, and strictly decodes the
|
||||
character-keyed typed schema before applying Narratio's party-domain checks.
|
||||
Successful metadata coordinates recover authored character-map order without a
|
||||
second YAML parse; class and alias slice order comes directly from typed decode.
|
||||
The canonical value retains a separate exact byte copy of its source so
|
||||
consumers can materialize the authored party document without reserializing it.
|
||||
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
|
||||
players-only projection.
|
||||
|
||||
An unversioned source is classified by the small legacy compatibility boundary
|
||||
in `party_legacy.go`; it deliberately exposes no parsed roster information.
|
||||
That boundary exists solely to isolate removable compatibility behavior from
|
||||
the canonical parser.
|
||||
An unversioned source is rejected before resolution, family expansion, or
|
||||
prepare-stage materialization.
|
||||
|
||||
## Validation Ownership
|
||||
|
||||
Configuration validation remains within `internal/config`, with the public
|
||||
entry points and pipeline-domain sequencing separated from the private owners
|
||||
for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
||||
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||
boundaries rather than extension points or a general validation framework.
|
||||
|
||||
Validation is read-only, fails fast, and does not define which of several
|
||||
independent invalid fields is reported first. Map-backed declarations are
|
||||
nevertheless traversed deterministically. Reusable domain normalization is an
|
||||
explicit preceding operation for both loaded and programmatically constructed
|
||||
pipelines. Notarius normalization commits canonical scalar, reference, and output
|
||||
values only after the complete normalization succeeds; its validator then checks
|
||||
that candidate without changing it. Scriptorium validates artifact declarations
|
||||
and outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||
dependency graph in deterministic artifact order.
|
||||
|
||||
Diagnostics identify the relevant semantic field. Merge conflicts also
|
||||
retain every claiming source needed to correct an additive ownership conflict.
|
||||
Complete prose, punctuation, formatting, and private validation choreography are
|
||||
not compatibility contracts.
|
||||
|
||||
## Diagnostics And Runtime Metadata
|
||||
|
||||
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
||||
Syntax, duplicate-key, merge, profile, and schema failures include the
|
||||
relevant source name and full field path. Additive conflicts report every
|
||||
claiming source so operators can repair the split without repeatedly
|
||||
rediscovering additional conflicts.
|
||||
|
||||
The loaded pipeline retains private runtime metadata for the absolute root
|
||||
path, ordered imports, selected profile name and selection source, selected
|
||||
overlay, contributing sources, effective digest, and leaf ownership. Base
|
||||
overlay, original ConfigKit metadata and source snapshots, effective digest,
|
||||
and leaf ownership. Base
|
||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
||||
metadata does not participate in YAML decoding or alter the public
|
||||
configuration model.
|
||||
|
||||
The effective digest is SHA-256 over deterministic canonical YAML produced from
|
||||
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
|
||||
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
|
||||
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
|
||||
execution, but the digest substitutes their normalized logical values captured
|
||||
before root-relative resolution, so relocating an equivalent configuration
|
||||
bundle does not change provenance. Because composition and resolution metadata
|
||||
bundle does not change provenance. Because ConfigKit and resolution metadata
|
||||
are private, the digest excludes source layout, profile name, and ownership.
|
||||
Configuration stores environment variable names rather than resolving raw
|
||||
credentials, so raw secret values are neither loaded nor hashed.
|
||||
Configuration stores credentials as ConfigKit typed references. A successful
|
||||
load resolves their values inside the opaque typed boundary, while this safe
|
||||
projection serializes only `EnvName()` identities. Credential-value changes are
|
||||
therefore never hashed; reference-name changes remain ordinary provenance.
|
||||
The optional Notarius credential follows the same path even for disabled or
|
||||
resumable extraction; normalization copies the opaque reference without reading
|
||||
or resolving it again. Direct YAML serialization has a configuration-owned safe
|
||||
projection so only the reference name can leave this boundary.
|
||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||
for later runtime expansion.
|
||||
|
||||
## Test Surfaces
|
||||
|
||||
`composition_test.go` protects the presence and merge algebra independently of
|
||||
the public schema. `pipeline_composition_test.go` exercises explicit imports,
|
||||
confinement, conflicts, strict decoding, metadata, and root-relative path
|
||||
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
||||
all-overlay validation, overlay behavior, provenance, option propagation, and
|
||||
effective-digest stability. Application configuration-loader tests protect the
|
||||
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
|
||||
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
||||
unique artifact/family base-declaration rule. Existing focused configuration
|
||||
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||
single-read boundary by changing the pipeline file after its initial load and
|
||||
confirming local session resolution retains the original pipeline. Other
|
||||
configuration tests continue to protect defaults and validation after assembly.
|
||||
`party_test.go` protects the versioned party schema, domain invariants, and
|
||||
deterministic players projection without involving campaign or runtime wiring.
|
||||
`party_test.go` protects the versioned typed party schema, domain invariants,
|
||||
authored character order, retained bytes, and deterministic players projection
|
||||
without involving campaign or runtime wiring.
|
||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||
input restrictions, legacy overrides, source provenance, and virtual players
|
||||
input selection.
|
||||
input restrictions, source provenance, and virtual players input selection.
|
||||
|
||||
## Artifact Family Resolution
|
||||
|
||||
|
||||
@@ -1,71 +1,55 @@
|
||||
# Internal: File Operations
|
||||
|
||||
`internal/fileops` owns the narrow mechanics for durable replacement of one
|
||||
`internal/fileops` owns the narrow mechanics for atomically visible replacement of one
|
||||
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||
and destination-directory policy.
|
||||
|
||||
## Destination Confinement
|
||||
## Destination Paths
|
||||
|
||||
Before it creates, replaces, or installs a destination file, `fileops` opens
|
||||
each ancestor from the filesystem root and rejects symbolic links or components
|
||||
that change during traversal. The resulting parent-directory handle is retained
|
||||
for sibling temporary-file creation and rename, so a later pathname swap cannot
|
||||
redirect the replacement. Existing destination symlinks are replaced as leaf
|
||||
entries; their targets are never followed.
|
||||
Ordinary workspace replacement uses standard operating-system path semantics.
|
||||
Callers with a root-scoped safety contract own that validation at their boundary.
|
||||
|
||||
Remote object acquisition uses a writer supplied by the storage owner. The
|
||||
writer receives a `fileops`-owned, already-open sibling temporary file rather
|
||||
than a mutable destination path. Callers still own remote object selection,
|
||||
validation, conflict handling, and final mode.
|
||||
Remote object acquisition is owned by its domain caller. The caller writes a
|
||||
sibling temporary file, validates the complete result, and passes that file to
|
||||
`InstallSiblingTempFile` only when it is ready to become canonical. Callers own
|
||||
remote object selection, validation, conflict handling, and temporary-file
|
||||
cleanup.
|
||||
|
||||
Directory promotion keeps the verified destination parent open while it creates
|
||||
the temporary tree, copies regular source entries, and performs the platform
|
||||
no-replace rename. Platforms without a verified handle-relative atomic
|
||||
no-replace primitive reject promotion before writing a temporary tree.
|
||||
Directory bundles are not promoted by `internal/fileops`; extraction owns its
|
||||
validated Notarius bundle rename.
|
||||
|
||||
## Cleanup Contract
|
||||
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
|
||||
the root and each target ancestor without following symlinks, then removes the
|
||||
tree through those directory handles. It rejects root deletion and any symlink
|
||||
encountered in the target path or tree; repeated removal of a missing target is
|
||||
successful. Command and post-publish policy remains owned by `internal/app`.
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant, opens
|
||||
that root with `os.OpenRoot`, and removes the normalized root-relative target
|
||||
with `Root.RemoveAll`. It rejects root, traversal, and out-of-root targets; a
|
||||
missing target is successful. `RemoveFileUnderRoot` additionally requires an
|
||||
existing leaf to be regular. Command and post-publish policy remains owned by
|
||||
`internal/app`.
|
||||
|
||||
## Confined Reads
|
||||
## Bounded Reads
|
||||
|
||||
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
|
||||
caller-selected root and relative file path; `ReadRegularFile` is its
|
||||
path-based convenience wrapper. They verify every ancestor through directory
|
||||
handles and admit only a stable regular-file handle. Callers enforce their own
|
||||
byte limits and access policy. Credential mode policy and environment
|
||||
precedence remain owned by `internal/app`.
|
||||
`ReadFileBounded` opens a normal path and reads at most its caller-supplied
|
||||
limit. Credential mode policy and environment precedence remain owned by
|
||||
`internal/app`.
|
||||
|
||||
## Replacement Contract
|
||||
|
||||
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||
sibling temporary file, writes the complete byte sequence, applies the
|
||||
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
|
||||
check, replaces the destination with a rename, then syncs the containing
|
||||
directory.
|
||||
caller-supplied mode, closes the file, runs an optional pre-rename check, and
|
||||
replaces the destination with a rename.
|
||||
|
||||
The pre-rename check is the last point at which a caller can cancel without
|
||||
installing a new destination. A failure before the rename leaves the old
|
||||
destination unchanged and removes the temporary file; any cleanup failure is
|
||||
returned alongside the primary failure. A failure after the rename may leave
|
||||
the new file visible, but it is not reported as crash-durable.
|
||||
destination unchanged and removes the temporary file. Successful replacement
|
||||
provides atomic visibility; it makes no crash- or power-loss durability claim.
|
||||
|
||||
Replacement follows the operating system's same-filesystem rename semantics.
|
||||
If a platform cannot replace an existing destination, the operation returns an
|
||||
error and never removes the old file as an emulation step.
|
||||
|
||||
## Directory-Sync Support
|
||||
|
||||
Linux and macOS attempt to sync the destination directory. Windows opens the
|
||||
directory with backup semantics and flushes its buffers. If either operation
|
||||
is unavailable for the platform, directory handle, or filesystem,
|
||||
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
|
||||
as successful crash-durable replacement.
|
||||
|
||||
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
|
||||
retain their compatibility behavior of creating the destination parent with
|
||||
the repository's workspace permissions before using this contract.
|
||||
`WriteFileAtomic` and the copy helpers create the destination parent with the
|
||||
repository's workspace permissions. `InstallSiblingTempFile` requires the
|
||||
caller to provide an existing temporary file in the existing destination
|
||||
directory.
|
||||
|
||||
@@ -2,29 +2,38 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the session-progress and invocation-audit models implemented by
|
||||
`internal/manifest`. Physical manifest placement belongs in
|
||||
Explain the session-progress model implemented by `internal/manifest`. Physical
|
||||
manifest placement belongs in
|
||||
[Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Session Manifest
|
||||
|
||||
`manifest.Manifest` records:
|
||||
|
||||
- required `schema_version: narratio.session-manifest.v1`
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map
|
||||
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
||||
remote commit identity, and each exact root-confined local target to its
|
||||
completion evidence
|
||||
- per-stage `stages` map, including the ordinary cleanup stage record
|
||||
|
||||
Session, campaign, and run identities in local and downloaded manifests must be
|
||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||
guidance rather than being normalized into a different workspace or remote
|
||||
namespace.
|
||||
|
||||
Decoding is strict at every typed manifest field. Unknown fields are rejected;
|
||||
the schema version remains `narratio.session-manifest.v1` and there is no
|
||||
compatibility reader that silently drops retired state. Cleanup progress uses
|
||||
only the cleanup entry in `stages`.
|
||||
|
||||
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
||||
non-nil and its embedded `name` exactly matches its map key; loading never
|
||||
invents either marker. A missing or unsupported schema version, or incomplete
|
||||
stage identity, is unsupported local state: move it aside and regenerate it,
|
||||
or restore a verified current committed snapshot when that is authoritative.
|
||||
|
||||
Prepare records independent `party` and `players` input checksums. In canonical
|
||||
party mode, the party record retains its campaign source identity while the
|
||||
players record uses `derived_from_party`; raw roster content is never embedded
|
||||
@@ -43,11 +52,11 @@ The model admits these stage states:
|
||||
|
||||
### Analyze-owned artifact state
|
||||
|
||||
The `analyze` stage record may carry `analyze_state_version: 1` and an
|
||||
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
||||
`analyze_artifacts` map keyed by normalized configured artifact key. The
|
||||
version is the authority marker: version 1 with no entries is a valid evaluated
|
||||
empty set, while an absent version is legacy aggregate-only state and provides
|
||||
no current configured-artifact evidence.
|
||||
empty set. An absent or unsupported version is unsupported local state and is
|
||||
rejected rather than being treated as aggregate-only authority.
|
||||
|
||||
Each analyze artifact record has one disposition:
|
||||
|
||||
@@ -72,18 +81,15 @@ do not replace the concrete configured key or infer family membership from a
|
||||
name, so older records without them remain valid.
|
||||
|
||||
The session-stage collection is the reconciled authority across invocations.
|
||||
The corresponding collection on an invocation's `analyze` stage record is an
|
||||
audit of only the artifacts evaluated or attempted by that run. These records
|
||||
remain analyze-owned data inside the fixed stage; they are not dynamic stages
|
||||
or generic subtasks.
|
||||
These records remain analyze-owned data inside the fixed stage; they are not
|
||||
dynamic stages or generic subtasks.
|
||||
|
||||
The stage result contract has one analyze-specific projection boundary. On
|
||||
success, the runner validates and deep-copies the complete reconciled session
|
||||
collection and the invocation subset. Aggregate session outputs are rebuilt in
|
||||
configured-key order from current session records only; invocation outputs are
|
||||
limited to current records produced by that invocation's run ID. Ordinary
|
||||
stage outputs cannot accompany this projection, so there is one source of
|
||||
artifact authority.
|
||||
Analyze checkpoints one complete validated candidate collection after each
|
||||
successful artifact and before exposing that artifact to later work. It saves a
|
||||
detached manifest candidate and installs its version, collection, and updated
|
||||
timestamp into live state only after the store reports success. Aggregate stage
|
||||
outputs are ordinary result outputs rebuilt in configured-key order from current
|
||||
session records only.
|
||||
|
||||
Successful incremental execution replaces only evaluated artifact records and
|
||||
preserves valid unrelated current records. Rebuilt outputs are compared by
|
||||
@@ -92,43 +98,31 @@ the same recomputed fingerprint to remain current, while a changed identity
|
||||
removes output authority from every unselected transitive dependent by marking
|
||||
it stale. A partial analyze invocation can therefore succeed while unrelated
|
||||
configured records remain stale. Existing canonical files never create current
|
||||
records without validated execution and projection.
|
||||
records without validated execution and a successful checkpoint.
|
||||
|
||||
Aggregate analyze status is deliberately coarser than this collection. Resume
|
||||
validation may skip a succeeded aggregate record when the selected artifact
|
||||
closure is current even if unrelated records are stale. Conversely, a stale
|
||||
aggregate record may cross the ordinary runner boundary and perform zero
|
||||
Scriptorium calls when reconciliation proves every selected artifact current;
|
||||
the successful projection then restores the aggregate status.
|
||||
ordinary successful stage completion then restores the aggregate status.
|
||||
|
||||
Analyze may return a projection together with an error. That restricted result
|
||||
cannot carry ordinary outputs, skip state, aggregate logs, generated configs,
|
||||
or metadata. The runner persists only the validated per-artifact collections,
|
||||
then marks the aggregate analyze and run state failed and invalidates delivery
|
||||
dependents conservatively. Unrelated current records survive because the
|
||||
session projection is complete. A malformed projection is not applied, and a
|
||||
failed session projection save restores the prior per-artifact authority before
|
||||
terminal failure persistence.
|
||||
On an artifact error, analyze installs a complete retry collection on the live
|
||||
manifest before returning. The active record is failed without output, current
|
||||
transitive dependents are stale, unrelated current records survive, and only
|
||||
earlier checkpointed completions remain current. The runner's ordinary failure
|
||||
path marks aggregate analyze failed, clears aggregate outputs, invalidates
|
||||
delivery dependents, and saves that collection. A terminal session-save failure
|
||||
is joined with the original adapter, validation, filesystem, cancellation, or
|
||||
checkpoint cause.
|
||||
|
||||
The incremental executor constructs this restricted projection at each
|
||||
scheduled artifact boundary. The active record is failed without output,
|
||||
current transitive dependents are stale, unrelated current records survive, and
|
||||
only earlier validated and materialized completions remain current in the
|
||||
invocation subset. Session failure state is persisted before invocation failure
|
||||
state. If either terminal save fails, its persistence error is joined with the
|
||||
original adapter, validation, or filesystem cause; a failed projection save
|
||||
does not turn incidental canonical bytes into manifest authority.
|
||||
|
||||
## Run Manifest
|
||||
|
||||
`manifest.RunManifest` is created for each invocation and records:
|
||||
|
||||
- invocation identity and `force` flag
|
||||
- the selected profile (when any) and secret-free effective configuration digest
|
||||
- requested stages
|
||||
- per-stage action (`run` or `skip`)
|
||||
- per-stage status
|
||||
- overall run status (`running`, `succeeded`, `failed`)
|
||||
If an artifact checkpoint fails, analyze rolls its working collection back to
|
||||
the last committed collection before constructing the active failed record.
|
||||
Newly materialized canonical bytes therefore remain incidental and cannot
|
||||
become current authority. A later ordinary run reconciles the durable records,
|
||||
reuses earlier current artifacts, and resumes with the failed or stale work.
|
||||
The same recovery applies after process interruption: the aggregate record may
|
||||
be non-successful, but every completed artifact checkpoint remains reusable.
|
||||
|
||||
## Remote Commit Manifest
|
||||
|
||||
@@ -145,38 +139,51 @@ binds that object by checksum, size, and generation. Readers strictly reject
|
||||
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||
objects that do not match their declaration.
|
||||
|
||||
The reader retains a temporary, clearly isolated compatibility path for a
|
||||
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
|
||||
path is removable after migration and is never used to write new state.
|
||||
An absent commit pointer means no remote state is published. Republish to
|
||||
establish a current immutable snapshot.
|
||||
|
||||
## Persistence Semantics
|
||||
|
||||
`manifest.LocalStore`:
|
||||
|
||||
- validates loaded documents;
|
||||
- normalizes missing maps/stage records;
|
||||
- writes through a sibling temporary file, syncing the completed file and
|
||||
destination directory after atomic replacement;
|
||||
- accepts only the exact supported session-manifest schema version;
|
||||
- rejects unknown fields and multiple JSON values;
|
||||
- validates stage-map identity without synthesizing missing maps, records, or
|
||||
names;
|
||||
- retains only representation-preserving collection normalization;
|
||||
- writes through a sibling temporary file and atomic replacement visibility;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
If the operating system or filesystem cannot sync a directory, save returns an
|
||||
explicit error instead of claiming crash-durable replacement. A returned error
|
||||
after the rename can therefore leave the new manifest visible but not confirmed
|
||||
durable; callers must reload it before retrying.
|
||||
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
||||
durability guarantee. Callers reload after a returned save error before retrying.
|
||||
An analyze checkpoint error leaves the prior manifest as per-artifact authority.
|
||||
The runner then attempts to persist the bounded failed/stale retry collection;
|
||||
if that terminal save also fails, its error is joined with the originating
|
||||
failure and the next invocation must reconcile from the last readable manifest.
|
||||
|
||||
## Execution Semantics
|
||||
|
||||
The application runner marks an executing stage running and then succeeded or
|
||||
failed in both manifests, persisting each transition. On success it records
|
||||
outputs, logs, generated configuration references, metadata, and—when the
|
||||
stage implements the optional contract—a versioned semantic-configuration
|
||||
fingerprint. Artifact
|
||||
Planning and execution share one pure pre-execution decision over the recorded
|
||||
stage status, force selection, and normalized resume-validation result. That
|
||||
decision identifies run versus reuse and any required stale and dependent
|
||||
invalidation transition. Planning applies it only to a cloned manifest.
|
||||
Execution remains the sole owner of observed stage results and persistence.
|
||||
|
||||
For each selected stage, the application runner computes semantic evidence,
|
||||
performs any eligible resume validation, applies the pre-execution decision,
|
||||
and persists required invalidation before either reusing the existing success
|
||||
or beginning an attempt. The same one-stage execution owner marks the attempt
|
||||
running before invoking the stage and then records and persists its failed,
|
||||
skipped, or successful terminal state before a later stage may start. On
|
||||
success it records outputs, logs, generated configuration references, metadata,
|
||||
and—when the stage implements the optional contract—a versioned
|
||||
semantic-configuration fingerprint. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded transitive dependent session-stage records stale.
|
||||
The application owns a fixed dependency relation distinct from execution order;
|
||||
dependents are returned in canonical order. Render and extract therefore never
|
||||
stale one another, while either can stale analyze, publish, and notify.
|
||||
stale one another, while either can stale analyze, publish, notify, and cleanup.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, metadata, and semantic fingerprint.
|
||||
@@ -184,8 +191,7 @@ Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins. Invocation run manifests remain immutable audit
|
||||
records of their own outcomes.
|
||||
before execution begins.
|
||||
|
||||
Aggregate lifecycle clearing deliberately preserves the analyze-owned
|
||||
per-artifact collection. This lets later reconciliation replace only evaluated
|
||||
@@ -194,30 +200,27 @@ existing aggregate-only lifecycle behavior and are forbidden from carrying the
|
||||
analyze-specific fields.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in both manifests, clears older outputs for the
|
||||
session-stage record along with older logs, generated configuration references,
|
||||
and metadata, then applies any bounded details from the current skip and
|
||||
continues. This self-skip is distinct from deciding not to execute an
|
||||
runner persists that outcome in the session manifest, clears older outputs,
|
||||
logs, generated configuration references, and metadata, then applies any
|
||||
bounded details from the current skip and continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs. An intentional self-skip records the current semantic
|
||||
fingerprint because it is a completed, reusable stage result; failed or
|
||||
interrupted work never promotes one.
|
||||
|
||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||
the `skip` action and reason. The session manifest deliberately retains its
|
||||
existing succeeded record because it remains the cross-invocation progress
|
||||
authority. If a stage supplies semantic configuration evidence, reuse first
|
||||
When an already-succeeded stage is skipped, the session manifest deliberately
|
||||
retains its existing succeeded record because it remains the cross-invocation
|
||||
progress authority. If a stage supplies semantic configuration evidence, reuse first
|
||||
requires the persisted positive schema version and lowercase SHA-256 digest to
|
||||
match the current resolved stage semantics. Missing legacy evidence, malformed
|
||||
match the current resolved stage semantics. Missing semantic evidence, malformed
|
||||
evidence, or a mismatch makes the stage and its fixed transitive dependents
|
||||
stale. The invocation skip copies the matched fingerprint for provenance but
|
||||
does not rewrite session authority. The existing stage-specific resume
|
||||
stale. The existing stage-specific resume
|
||||
validator runs only after this semantic check succeeds; both checks are
|
||||
required. Extraction and analyze have resume validators and may reject an
|
||||
otherwise eligible skip when their selected durable evidence is obsolete; the
|
||||
runner marks the aggregate record stale and executes it. Analyze's validator
|
||||
can still accept a partial selection when only unrelated artifact records are
|
||||
stale.
|
||||
required. Extraction, analyze, and cleanup have resume validators and may
|
||||
reject an otherwise eligible skip when their selected durable evidence is
|
||||
obsolete; the runner marks the aggregate record stale and executes it.
|
||||
Analyze's validator can still accept a partial selection when only unrelated
|
||||
artifact records are stale.
|
||||
|
||||
Implemented reuse coverage is deliberately split between aggregate semantic
|
||||
evidence and focused durable validators:
|
||||
@@ -235,6 +238,7 @@ evidence and focused durable validators:
|
||||
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
|
||||
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
|
||||
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
|
||||
| cleanup | aggregate semantic fingerprint plus publish-evidence and target-absence validator | [cleanup](stage-cleanup.md), [workspace](workspace.md) |
|
||||
|
||||
These contracts record resolved choices Narratio can observe, not operational
|
||||
runner tuning. External model, module, prompt, profile, and configuration-file
|
||||
@@ -242,10 +246,8 @@ contents that a tool privately loads remain outside the contract when their
|
||||
configured identifier is unchanged; operators must force the affected work
|
||||
after such a private content change.
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
|
||||
Both manifests retain the most recently resolved invocation's bounded
|
||||
The session manifest is the authoritative stage-progress ledger across
|
||||
invocations. It retains the most recently resolved invocation's bounded
|
||||
configuration provenance. It identifies the selected profile name and source
|
||||
(`default` or `cli`) plus the effective configuration digest, but never a raw
|
||||
secret or profile content. This provenance is informational: it does not
|
||||
@@ -254,9 +256,9 @@ invalidates only stages whose semantic configuration changed. When a private
|
||||
external-tool model, module, prompt, or profile changes behind an unchanged
|
||||
configured identifier, use `--force` for the affected work.
|
||||
|
||||
`session plan` computes the same current fingerprint and applies the same
|
||||
comparison and invalidation rules to a cloned manifest. It predicts the runner
|
||||
decision without persisting session or invocation state. The shared helper
|
||||
`session plan` computes the same current fingerprint and applies the shared
|
||||
decision and invalidation rules to a cloned manifest. It predicts the runner
|
||||
decision without persisting session state. The semantic fingerprint owner
|
||||
hashes deterministic JSON from stage-owned typed structs; stage providers must
|
||||
exclude secrets, complete effective-configuration dumps, and operational
|
||||
values that cannot affect canonical results. Concrete coverage is owned by the
|
||||
@@ -266,7 +268,7 @@ Before an explicitly bounded execution starts after `prepare`, the application
|
||||
reads the session manifest and accepts only `succeeded` or `skipped` for every
|
||||
excluded canonical prefix stage. The first other status or absent record fails
|
||||
the request before layout mutation, adapter initialization, session-manifest
|
||||
writes, or run-manifest creation. Excluded prefix records are not passed to
|
||||
writes, or run-directory creation. Excluded prefix records are not passed to
|
||||
resume validators. Records after the selected end are not prerequisites and
|
||||
may be made stale by selected work without being scheduled.
|
||||
|
||||
@@ -283,33 +285,30 @@ session manifest must agree on campaign and session identity before execution;
|
||||
the current projection is refreshed for every invocation while stage progress,
|
||||
inputs, and durable artifacts remain session history.
|
||||
|
||||
For handled failures after an invocation record is created, the runner records
|
||||
the failure on the session ledger and persists it before persisting the failed
|
||||
run audit record. This preserves the resume authority while making a partial
|
||||
persistence disagreement visible. Abrupt process death remains an accepted case
|
||||
where a durable running record can require operator interpretation.
|
||||
For handled failures after the session identity transition, the runner records
|
||||
the failure on the session ledger and atomically attempts its save while
|
||||
preserving the originating error. Abrupt process death remains an accepted case
|
||||
where the last session transition can require operator interpretation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- semantic fingerprint comparison precedes stage-specific resume validation.
|
||||
- only successful and intentional-skipped results promote current semantic
|
||||
evidence; invocation reuse copies evidence without replacing session state.
|
||||
evidence.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale succeeded stages in the fixed dependency relation.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
- the session manifest is the only durable local progress ledger.
|
||||
- remote commitment is established by a verified current pointer and remote
|
||||
commit relationship, never by a mutable session-manifest boolean.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Models and transitions: `internal/manifest/manifest.go`,
|
||||
`internal/manifest/run_manifest.go`
|
||||
- Models and transitions: `internal/manifest/manifest.go`
|
||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Persistence and validation: `internal/manifest/store.go`
|
||||
- Package tests: `internal/manifest/*_test.go`
|
||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||
|
||||
@@ -22,20 +22,29 @@ session safety controls, and runs commands. Pipeline commands execute the
|
||||
canonical stage sequence through adapter interfaces, while manifests record
|
||||
progress and artifact services resolve durable inputs and outputs.
|
||||
|
||||
The pipeline runner follows one explicit invocation lifecycle: inspect bounded
|
||||
prerequisites without mutation, prepare the workspace layout, acquire one
|
||||
session lock, load or create the authoritative session manifest, compose only
|
||||
the collaborators needed by the selected range, execute each selected stage in
|
||||
order, and then return the summary. Cleanup, when selected, is the ordinary
|
||||
final stage rather than runner finalization work.
|
||||
The lock remains held from manifest initialization through finalization, and a
|
||||
failed stage or required manifest save prevents later stages from starting.
|
||||
|
||||
## Components
|
||||
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
|
||||
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus atomic single-file replacement, bounded reads, workspace modes, and root-scoped cleanup helpers. |
|
||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||
|
||||
The application boundary composes concrete implementations. Stages depend on
|
||||
@@ -43,10 +52,20 @@ Narratio-level contracts; external transport and SDK details remain in
|
||||
adapters. The normative rules for these relationships remain in
|
||||
[Architecture](../policy/architecture.md).
|
||||
|
||||
Command execution normalizes one private dependency value per invocation.
|
||||
Pipeline configuration-path discovery, configuration reads, secret-directory
|
||||
reads, remote downloads, object-store construction, and restore phase functions
|
||||
use that value throughout the command. Public command entry points supply the
|
||||
canonical discovery and production defaults, while focused tests may provide
|
||||
invocation-owned discovery candidates or replace only the external or sequencing
|
||||
boundary they exercise without mutating package state.
|
||||
|
||||
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
||||
model. Planning clones session state and applies selected-stage transitions and
|
||||
resume validation in memory; it does not create invocation state or initialize
|
||||
stage-execution adapters. Command configuration loading can still retrieve a
|
||||
model and the same pure pre-execution decision over recorded status, force, and
|
||||
normalized resume validation. Planning applies that decision and modeled
|
||||
selected-stage transitions to cloned session state; it does not persist, create
|
||||
invocation state, initialize stage-execution adapters, or claim to observe an
|
||||
execution result. Command configuration loading can still retrieve a
|
||||
missing session file through configured remote storage. It retains the initially
|
||||
composed pipeline and selected campaign while resolving either a local or
|
||||
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
||||
@@ -68,6 +87,7 @@ The implemented canonical order is:
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (no-op)
|
||||
12. [`cleanup`](stage-cleanup.md)
|
||||
|
||||
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||
`noop` notification mode. Its versioned semantic evidence records that delivery
|
||||
@@ -81,18 +101,19 @@ registry owns the flat execution sequence. The application orchestration owner
|
||||
uses a fixed, validated dependency relation to find transitive dependents in
|
||||
canonical order. In particular, `render` and `extract` are sibling consumers of
|
||||
trimmed transcript state: neither invalidates the other, while either can stale
|
||||
`analyze`, `publish`, and `notify`.
|
||||
`analyze`, `publish`, `notify`, and `cleanup`. Cleanup is a dependent of notify,
|
||||
so any earlier replacement can stale a prior cleanup result.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Configuration Internals](configuration.md): pipeline composition, import
|
||||
confinement, field ownership, decoding, and root-relative path semantics.
|
||||
- [Configuration Internals](configuration.md): pipeline imports and profiles,
|
||||
field ownership, decoding, and root-relative path semantics.
|
||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
resolution, previous requirements, and current-state helpers.
|
||||
- [Manifest Internals](manifest.md): session and run records, persistence, and
|
||||
execution transitions.
|
||||
- [Manifest Internals](manifest.md): session records, persistence, and execution
|
||||
transitions.
|
||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||
guardrails.
|
||||
@@ -108,6 +129,7 @@ trimmed transcript state: neither invalidates the other, while either can stale
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
- [`cleanup`](stage-cleanup.md)
|
||||
|
||||
Use this map to find an owner, then read its focused documentation and tests
|
||||
before changing behavior.
|
||||
|
||||
@@ -10,7 +10,9 @@ dependency order, and safely materialize validated outputs.
|
||||
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
||||
party artifact families have already expanded into this map during
|
||||
configuration resolution, including corresponding member dependencies and
|
||||
rewritten member-artifact input sources
|
||||
rewritten member-artifact input sources. Each artifact's dependency list is
|
||||
already the normalized union of explicit ordering edges and configured-artifact
|
||||
input edges.
|
||||
- optional selected artifact keys supplied through the stage environment
|
||||
- built-in, configured, extraction, and previous-session source references in
|
||||
artifact inputs
|
||||
@@ -31,12 +33,14 @@ Supported source families:
|
||||
## Key Behavior
|
||||
|
||||
- when `pipeline.scriptorium` is absent or no configured artifact is
|
||||
executable, completes successfully with no outputs and records explanatory
|
||||
metadata. This is not an explicit self-skip: both manifests record success,
|
||||
satisfy publish's prerequisite, and an ordinary later run reuses the result
|
||||
while the effective set remains empty. Enabling or selecting an artifact
|
||||
later makes missing versioned evidence non-resumable and schedules it without
|
||||
requiring force.
|
||||
executable, completes successfully and records explanatory metadata. Before
|
||||
returning, it reconciles and checkpoints the complete session collection:
|
||||
records removed from configuration are dropped, while valid records for
|
||||
configured but unselected artifacts remain authoritative. This is not an
|
||||
explicit self-skip: the session manifest records success, satisfies publish's
|
||||
prerequisite, and an ordinary later run reuses the result while the effective
|
||||
set remains empty. Enabling or selecting an artifact later makes missing
|
||||
versioned evidence non-resumable and schedules it without requiring force.
|
||||
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||
and configured extraction lanes. Extraction availability is hydrated only
|
||||
from compatible successful extraction evidence.
|
||||
@@ -49,6 +53,9 @@ Supported source families:
|
||||
manifest record and durable output pass the configured-artifact evidence
|
||||
contract; an incidental or stale canonical file is unavailable.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- consumes only the concrete dependency graph finalized by configuration;
|
||||
planning, prerequisite closure, execution, reuse, invalidation, and semantic
|
||||
evidence do not infer a second graph from inputs.
|
||||
- resolves required/optional inputs per artifact source definition into an
|
||||
ordered semantic identity. Each identity records the configured input name,
|
||||
canonical source ID, required policy, explicit presence, source contract,
|
||||
@@ -75,7 +82,7 @@ Supported source families:
|
||||
validated current dependency-output identities, and sorted effective
|
||||
Scriptorium variables (including Narratio's sticky session variable).
|
||||
- provides read-only reconciliation that classifies each configured record as
|
||||
current, stale, missing, failed, legacy, or otherwise non-resumable, and
|
||||
current, stale, missing, failed, or otherwise non-resumable, and
|
||||
separately identifies manifest records removed from current configuration.
|
||||
A record is current only when its fingerprint version and value match and its
|
||||
configured output still passes manifest-authoritative evidence validation.
|
||||
@@ -89,13 +96,14 @@ Supported source families:
|
||||
rebuilt when required, while unrelated disabled artifacts are excluded.
|
||||
- the work plan carries explicit targets, prerequisite-only work, deterministic
|
||||
execution and reuse lists, invalidated and removed records, and a cloned
|
||||
projected record collection. Valid unrelated configured records survive the
|
||||
projection, removed records are omitted, and legacy files never become
|
||||
current without regeneration.
|
||||
reconciled starting collection. Valid unrelated configured records survive
|
||||
reconciliation, removed records are omitted, and incidental files never
|
||||
become current without manifest-backed regeneration.
|
||||
- implements aggregate resume validation by running the same read-only catalog,
|
||||
fingerprint reconciliation, and work planner used by execution. A succeeded
|
||||
aggregate record is reusable exactly when the selected closure schedules no
|
||||
artifact work; stale unrelated records do not block a partial selection.
|
||||
artifact work and no removed configuration records require a checkpoint;
|
||||
stale unrelated records do not block a partial selection.
|
||||
- exposes the typed artifact decision to `session plan`. Planning applies it to
|
||||
a cloned manifest after modeling earlier selected stage transitions, so
|
||||
aggregate run/skip and artifact execute/reuse decisions match the ordinary
|
||||
@@ -104,6 +112,10 @@ Supported source families:
|
||||
prerequisites remain available through the runtime catalog without invoking
|
||||
Scriptorium; newly produced prerequisites enter that catalog with the same
|
||||
contract, checksum, and size identity used for persisted current evidence.
|
||||
- before the first adapter call, checkpoints the reconciled starting collection
|
||||
when it removed configuration records or cleared obsolete authority. After
|
||||
each successful artifact, checkpoints the complete collection before making
|
||||
that artifact available to the next scheduled dependent.
|
||||
- keeps adapter output in the invocation's run-local analyze directory until
|
||||
it is a safe, non-empty, bounded regular file with a calculated checksum and
|
||||
complete output contract. Canonical replacement uses the shared atomic file
|
||||
@@ -112,16 +124,15 @@ Supported source families:
|
||||
- records each successful artifact's freshly computed fingerprint, canonical
|
||||
relative output path, contract, checksum, size, producer run ID, bounded
|
||||
Scriptorium provenance, logs, and generated configuration references in the
|
||||
analyze-owned projection.
|
||||
analyze-owned manifest collection.
|
||||
- preserves valid unrelated current records during partial execution. If a
|
||||
rebuilt output's bytes and contract are unchanged, unselected dependents may
|
||||
remain current. If that semantic identity changes, unselected transitive
|
||||
dependents become stale without being executed; dependents included in the
|
||||
invocation are evaluated in dependency order instead.
|
||||
- reports all evaluated targets and prerequisites in invocation state. The
|
||||
runner reconstructs aggregate session outputs from every current session
|
||||
record and invocation outputs from only records produced by the current run.
|
||||
Unrelated stale records do not make an otherwise successful partial
|
||||
- returns aggregate outputs from every current session record in normalized-key
|
||||
order after the complete reconciled collection has been checkpointed;
|
||||
unrelated stale records do not make an otherwise successful partial
|
||||
invocation fail.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
@@ -137,25 +148,31 @@ Supported source families:
|
||||
guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
- a scheduled artifact failure returns the restricted analyze-state projection
|
||||
- a scheduled artifact failure installs a retry collection on the live manifest
|
||||
with the active artifact marked `failed`, a bounded error, and no output
|
||||
authority. Current transitive dependents become stale without execution.
|
||||
- earlier artifacts from the invocation remain current only after their
|
||||
run-local output passed validation and canonical materialization. They remain
|
||||
in invocation history; unattempted later artifacts do not appear there.
|
||||
- earlier artifacts remain current only after their run-local output passed
|
||||
validation, canonical materialization, and a successful complete-collection
|
||||
manifest checkpoint; unattempted later artifacts retain their reconciled
|
||||
session state.
|
||||
- unrelated current records survive a partial failure. Old canonical bytes for
|
||||
the failed artifact and newly materialized bytes whose projection cannot be
|
||||
persisted are incidental, not current evidence.
|
||||
- the runner persists a valid partial projection before it marks aggregate
|
||||
analyze failed and invalidates publish and notify through the application
|
||||
dependency relation. Projection-persistence errors retain the last durable
|
||||
per-artifact authority and are joined with the original failure context.
|
||||
the failed artifact and newly materialized bytes whose checkpoint fails are
|
||||
incidental, not current evidence.
|
||||
- a checkpoint failure stops the queue immediately. The in-memory retry
|
||||
collection starts from the last successful checkpoint, marks the active
|
||||
artifact failed without output authority, and stales its current transitive
|
||||
dependents.
|
||||
- the runner's ordinary error path marks aggregate analyze failed, clears its
|
||||
aggregate outputs, invalidates publish, notify, and cleanup through the
|
||||
application dependency relation, and persists the live retry collection.
|
||||
Terminal-save errors are joined with the original artifact or checkpoint
|
||||
failure.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- input-identity resolution is read-only: it does not invoke adapters,
|
||||
materialize outputs, update status, or create run records.
|
||||
materialize outputs, update status, or create run-local files.
|
||||
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
||||
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
||||
executable or private transitive configuration contents. A change that is
|
||||
@@ -164,6 +181,9 @@ Supported source families:
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
- a canonical file without current per-artifact manifest evidence is never
|
||||
promoted to current state.
|
||||
- an ordinary retry after interruption or partial failure reuses every valid
|
||||
earlier checkpoint and schedules only failed, stale, missing, or otherwise
|
||||
non-current work in dependency order.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
@@ -173,13 +193,18 @@ Supported source families:
|
||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||
- Implementation and tests: `internal/stage/analyze.go`,
|
||||
`internal/stage/analyze_input_identity.go`, `internal/stage/analyze_test.go`,
|
||||
`internal/stage/analyze_input_identity_test.go`,
|
||||
`internal/stage/analyze_artifact_execution.go`,
|
||||
`internal/stage/analyze_execution_state.go`,
|
||||
`internal/stage/analyze_input_identity.go`,
|
||||
`internal/stage/analyze_fingerprint.go`,
|
||||
`internal/stage/analyze_reconciliation.go`, `internal/stage/analyze_plan.go`,
|
||||
`internal/stage/analyze_resume.go`, `internal/stage/analyze_test.go`,
|
||||
`internal/stage/analyze_checkpoint_test.go`,
|
||||
`internal/stage/analyze_input_identity_test.go`,
|
||||
`internal/stage/analyze_fingerprint_test.go`,
|
||||
`internal/stage/analyze_reconciliation.go`, and
|
||||
`internal/stage/analyze_reconciliation_test.go`,
|
||||
`internal/stage/analyze_plan.go`, `internal/stage/analyze_plan_test.go`, and
|
||||
`internal/stage/analyze_incremental_execution_test.go`, and
|
||||
`internal/stage/analyze_plan_test.go`,
|
||||
`internal/stage/analyze_incremental_execution_test.go`,
|
||||
`internal/stage/analyze_failure_test.go`,
|
||||
`internal/stage/analyze_resume.go`, and `internal/stage/analyze_resume_test.go`
|
||||
`internal/stage/analyze_resume_test.go`, and
|
||||
`internal/app/analyze_checkpoint_recovery_test.go`.
|
||||
|
||||
54
docs/internal/stage-cleanup.md
Normal file
54
docs/internal/stage-cleanup.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# Stage: cleanup
|
||||
|
||||
## Purpose
|
||||
|
||||
Remove explicitly configured local data for the run selected by a successful
|
||||
publish commit. Cleanup is the twelfth and final canonical stage, after
|
||||
`notify`.
|
||||
|
||||
## Authorization And Targets
|
||||
|
||||
Cleanup is disabled unless at least one of these policies is true:
|
||||
|
||||
- `pipeline.spool.delete_audio_after_publish`
|
||||
- `pipeline.workspace.cleanup_after_publish`
|
||||
|
||||
An enabled cleanup requires the manifest's publish stage to be succeeded with
|
||||
`uploaded=true`, a canonical `published_run_id`, and the exact remote commit and
|
||||
current commit-pointer keys for the manifest campaign/session prefix. The
|
||||
configured campaign and session must match the manifest. Cleanup derives its
|
||||
only possible targets through the canonical artifact path helpers:
|
||||
|
||||
- the published run's session spool-audio directory; and
|
||||
- the published run's workspace run directory.
|
||||
|
||||
It does not scan parent directories, infer the newest run, or read object
|
||||
storage. Every deletion uses the root-confined file operation, which rejects a
|
||||
root itself, traversal, symlink escape, and out-of-root targets.
|
||||
|
||||
## Execution And Resume
|
||||
|
||||
Disabled cleanup records an explicit skipped result. Enabled cleanup removes
|
||||
targets in deterministic policy order and treats an already-missing target as
|
||||
success. A successful record contains the published run ID and completed policy
|
||||
names.
|
||||
|
||||
A previous cleanup success is reusable only while its semantic configuration
|
||||
matches and every selected target remains absent. A recreated target makes the
|
||||
stage non-resumable. A failure is recorded against cleanup; publish remains
|
||||
succeeded and remote current state is unchanged. On a later normal full run,
|
||||
valid upstream stages are skipped and cleanup is retried without republishing.
|
||||
|
||||
`narratio publish` selects publish only and never invokes cleanup. A bounded run
|
||||
through `notify` or any earlier endpoint also excludes it. Operators may run
|
||||
`narratio run-stage cleanup <session_id>` directly; missing, failed, malformed,
|
||||
or identity-mismatched publication evidence fails before deletion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Planning and evidence validation: `internal/stage/cleanup_plan.go`
|
||||
- Execution and resume: `internal/stage/cleanup.go`
|
||||
- Root-confined deletion: `internal/fileops/cleanup.go`
|
||||
- Focused tests: `internal/stage/cleanup_plan_test.go`,
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
@@ -19,12 +19,15 @@ procedures belong in [Operations](../operations.md).
|
||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||
2. resolves every configured prepared reference through the shared
|
||||
manifest-authoritative identity resolver before creating run-local output;
|
||||
3. streams each verified reference into an invocation-local snapshot and
|
||||
rejects any source change observed while copying;
|
||||
3. resolves each reference to a stable content-addressed session snapshot,
|
||||
reusing verified bytes or reconstructing a missing/corrupt regular entry,
|
||||
and rejects any source change observed while copying;
|
||||
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
||||
including sorted reference identities;
|
||||
5. creates a run-local staging directory and invokes the injected
|
||||
`notarius.Runner`;
|
||||
`notarius.Runner`, carrying the optional already-resolved credential
|
||||
reference without exposing its value, plus effective resume and chunk-cache
|
||||
controls after force precedence;
|
||||
6. revalidates the reference snapshots, then validates the v2 successful
|
||||
receipt, confined index, management documents, configured required lane
|
||||
descriptors, validation summaries, and regular payload files;
|
||||
@@ -41,13 +44,21 @@ Narratio run ID, the resolved trimmed-input identity, and invocation
|
||||
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||
source ID, producer stage/output/run identity, and resolution provenance.
|
||||
Reference metadata contains only selector, source ID, canonical session-relative
|
||||
path, checksum, and size; adapter requests receive selector and absolute
|
||||
invocation-local snapshot path, never payload contents. Snapshot bytes must
|
||||
path, checksum, and size; adapter requests receive selector and absolute stable
|
||||
snapshot path, never payload contents. Snapshot bytes must
|
||||
match the prepared identity both before and after Notarius runs, so a concurrent
|
||||
prepared-file replacement cannot make recorded provenance describe different
|
||||
bytes from those supplied to Notarius.
|
||||
Validation completes before
|
||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||
Validation completes before the single durable rename, so a rejected result
|
||||
cannot expose a partial durable bundle.
|
||||
|
||||
Snapshot destinations are owned by `internal/artifacts` under
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
|
||||
or timestamp and can be reused across invocations. Identical bytes and basename
|
||||
share a verified file; changed bytes select a different digest directory.
|
||||
Unsafe path components and non-regular conflicts fail without recursive
|
||||
deletion. Snapshot paths are delivery details and never become manifest output
|
||||
evidence or prerequisites for reuse of an already complete extract bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded analysis and delivery dependents stale. Render is an independent
|
||||
@@ -61,6 +72,10 @@ semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
||||
reference selector/source mappings, sorted declared output contracts, and each
|
||||
canonical `narratio.extraction.<key>` output identity. It excludes executable,
|
||||
timeout, working directory, config path, and private Notarius config contents.
|
||||
Credential presence, reference name, and resolved value are likewise
|
||||
operational-only and excluded. Ordinary command loading still resolves a
|
||||
selected reference before resume evaluation, but a credential-only change does
|
||||
not rerun a valid completed extraction or stale its dependents.
|
||||
|
||||
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
||||
stage record still matches the current byte- and provenance-bearing invocation
|
||||
@@ -69,7 +84,10 @@ sorted prepared-reference identities, pipeline identity, and configured output
|
||||
contracts. The same reference helper and transcript identity are resolved again
|
||||
for artifact evidence, so changing current transcript bytes, reference bytes,
|
||||
or producer identity makes the prior extraction obsolete. Operational runner
|
||||
settings do not invalidate otherwise current durable evidence.
|
||||
settings do not invalidate otherwise current durable evidence. This includes
|
||||
`resume` and `chunk_cache`; changing them alone cannot make a complete
|
||||
extraction non-resumable. If extraction executes, ordinary runs forward the
|
||||
configured values, while force disables resume and selects `refresh`.
|
||||
|
||||
A valid prepared-reference change makes extraction non-resumable. Missing,
|
||||
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
||||
@@ -92,17 +110,17 @@ identifier.
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, inconsistent warning or diagnostic envelopes,
|
||||
required-lane rejection or incomplete validation, payload inspection,
|
||||
checksum, or promotion errors fail the stage through ordinary manifest
|
||||
checksum, or installation errors fail the stage through ordinary manifest
|
||||
transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and promotion.
|
||||
has passed validation and installation.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier invocation manifest and immutable promoted bundle remain
|
||||
available for audit and recovery.
|
||||
while the earlier immutable promoted bundle remains available for audit and
|
||||
recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
@@ -110,9 +128,13 @@ available for audit and recovery.
|
||||
`internal/stage/extract_resume.go`,
|
||||
`internal/stage/extract_test.go`,
|
||||
`internal/stage/semantic_contracts_delivery.go`
|
||||
- Stable snapshot materialization and verification:
|
||||
`internal/stage/extract_references.go`
|
||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||
`internal/adapters/notarius/subprocess_test.go`
|
||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`
|
||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||
- Opt-in Notarius v0.7.0 production-boundary coverage:
|
||||
`internal/stage/notarius_cache_integration_test.go`
|
||||
|
||||
@@ -15,7 +15,7 @@ Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||
- resolves polished transcript only from verified manifest output evidence.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
- validates normalized transcript and optional report.
|
||||
|
||||
@@ -16,7 +16,7 @@ Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves base transcript from merge outputs/canonical fallback.
|
||||
- resolves base transcript only from verified merge manifest output evidence.
|
||||
- invokes an Audita runner configured with static model/runtime options; the
|
||||
invocation supplies paths and modules.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
@@ -35,8 +35,9 @@ validation model, module set, transcript description, output schema, selected
|
||||
external configuration path, and canonical polished-transcript identity. Module
|
||||
ordering is normalized because the configured modules form a set. Audita's
|
||||
executable, timeouts, concurrency, report and debug behavior, work retention,
|
||||
and credential environment name are operational and do not invalidate a
|
||||
successful result.
|
||||
and resolved credential value are operational and do not invalidate a
|
||||
successful result. The public credential reference name may participate in
|
||||
configuration provenance but never exposes its resolved value.
|
||||
|
||||
Narratio can fingerprint a selected model, module, or configuration identifier,
|
||||
but it cannot inspect content that Audita privately resolves behind that stable
|
||||
|
||||
@@ -39,9 +39,8 @@ Materialize canonical current-session inputs before processing stages.
|
||||
- materializes a configured spell catalog with checksum and provenance, or
|
||||
safely removes an obsolete canonical spell catalog and its manifest record
|
||||
when the effective input is omitted.
|
||||
- in canonical party mode, copies the validated raw party bytes unchanged and
|
||||
deterministically generates the prepared players projection; legacy mode
|
||||
continues to copy its opaque party and explicit players sources.
|
||||
- copies the validated canonical party bytes unchanged and deterministically
|
||||
generates the prepared players projection.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||
- resolves the pointer-selected previous source through the shared resolver;
|
||||
@@ -49,8 +48,10 @@ Materialize canonical current-session inputs before processing stages.
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||
are typed skipped results. Committed sources use their exact source-to-destination
|
||||
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
|
||||
are typed skipped results. Each requirement resolves only from its unique
|
||||
`published_output` declaration in the pointer-selected remote commit. Prepare
|
||||
strips that selected run's immutable `published/` prefix to derive the local
|
||||
cache path, then validates the committed checksum, size, and generation.
|
||||
|
||||
## Invariants
|
||||
|
||||
|
||||
@@ -7,15 +7,16 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
||||
## Inputs
|
||||
|
||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||
- invocation-scoped run files
|
||||
- declared run-local diagnostic files
|
||||
- resolved publish output rules
|
||||
- effective publish locks (static + remote merged lock set), revalidated at the
|
||||
remote commit boundary
|
||||
- independent per-source remote lock objects read at planning start
|
||||
- the validated prior current commit when an initially locked source may carry
|
||||
- durable previous-session cache files when present
|
||||
|
||||
## Outputs
|
||||
|
||||
- uploaded invocation record and selected publish outputs;
|
||||
- uploaded session manifest, declared run-local diagnostics, and selected
|
||||
publish outputs;
|
||||
- uploaded durable previous-session cache files when present;
|
||||
- immutable run-scoped commit manifest; and
|
||||
- current commit pointer, written last.
|
||||
@@ -25,14 +26,27 @@ Exact remote placement and the operator workflow belong in
|
||||
|
||||
## Key Behavior
|
||||
|
||||
`publish.go` retains the transaction-owning `Run` path. Source discovery,
|
||||
manifest-derived archive metadata, confined opening, and checksum validation
|
||||
live in `publish_sources.go`; artifact-catalog matching, selection, required
|
||||
and optional rules, destination resolution, and collision inputs live in
|
||||
`publish_outputs.go`. Copying, hashing, immutable upload, reuse checks, and
|
||||
remote verification live in `publish_transfer.go`; committed-manifest,
|
||||
immutable-commit, and current-pointer encoding and writes live in
|
||||
`publish_commit.go`. `publish_output_plan.go` owns the deterministic, I/O-free
|
||||
five-outcome model and its secret-free metadata projection.
|
||||
|
||||
- when publishing or run upload is disabled, completes successfully with no
|
||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||
both manifests record success. Enablement and upload policy are fingerprinted,
|
||||
so changing either automatically makes the prior result non-resumable.
|
||||
the session manifest records success. Enablement and upload policy are
|
||||
fingerprinted, so changing either automatically makes the prior result
|
||||
non-resumable.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- derives a deterministic run-archive allowlist from the validated run
|
||||
`manifest.json`: declared run-local outputs, logs, generated configs, and the
|
||||
manifest itself. Unlisted workspace files are not archive candidates.
|
||||
- derives a deterministic run-archive allowlist from the session manifest's
|
||||
declared current-run outputs, logs, and generated configurations. References
|
||||
outside the current run root, including canonical session files and prior-run
|
||||
diagnostics, are not archive candidates. No distinguished diagnostic file is
|
||||
read or uploaded.
|
||||
- opens each archive candidate beneath its archive root without following
|
||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||
checks a declared checksum when present, then streams the opened descriptor.
|
||||
@@ -41,26 +55,34 @@ Exact remote placement and the operator workflow belong in
|
||||
- resolves publish output sources through runtime artifact catalog and
|
||||
manifest-aware resolution. Configured Scriptorium outputs are publishable
|
||||
only from validated `current` per-artifact analyze evidence; an incidental
|
||||
canonical file, legacy aggregate output, stale/failed/unselected record, or
|
||||
mismatched path, size, or checksum remains unavailable. This does not change
|
||||
the explicit compatibility policies owned by built-in, extraction, or
|
||||
previous-session sources.
|
||||
canonical file, aggregate output, stale/failed/unselected record, or
|
||||
mismatched path, size, or checksum remains unavailable. Built-in and
|
||||
extraction sources likewise require their own current manifest evidence.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
- plans every configured output into exactly one source-sorted outcome:
|
||||
`published`, `carried_locked`, `locked_unpublished`, `optional_missing`, or
|
||||
`unselected`.
|
||||
- an initially locked selected output carries its exact prior
|
||||
`published_output` declaration when one exists. The declaration may point to
|
||||
an earlier run in the same session and retains its source, key, checksum,
|
||||
size, and generation; the bytes are not copied.
|
||||
- verifies a carried object's exact key, size, and generation through object
|
||||
metadata without downloading or opening the local output source.
|
||||
- an initially locked source with no prior declaration remains absent even
|
||||
when the output rule is required. Optional missing unlocked outputs remain
|
||||
absent; required missing unlocked outputs fail.
|
||||
- creates one complete immutable source-to-destination mapping before upload;
|
||||
- uploads and verifies every declared immutable object and the commit manifest;
|
||||
- updates `current/commit-pointer.json` exactly once, last; and
|
||||
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
|
||||
- rechecks remote lock state immediately before the pointer update. A newly
|
||||
committed lock aborts selection, leaving any uploaded immutable attempt
|
||||
unselected.
|
||||
- reads the mutable remote lock document through a direct limit-plus-one read
|
||||
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
|
||||
with the opened body for conditional updates. Oversized lock documents fail
|
||||
before YAML decoding; published artifact payloads do not use this limit.
|
||||
- writes the immutable commit from the combined new and exact carried
|
||||
declarations.
|
||||
- rereads every initially unlocked participating source after the immutable
|
||||
commit. A newly present lock aborts selection, leaving the uploaded immutable
|
||||
attempt unselected. Removing an initial lock may conservatively produce one
|
||||
carry because the initial snapshot controls the plan.
|
||||
- updates `current/commit-pointer.json` exactly once, last, and writes no other
|
||||
mutable publication control record.
|
||||
|
||||
## Metadata Signals
|
||||
|
||||
@@ -68,9 +90,8 @@ Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
- previous uploads
|
||||
- skipped optional outputs
|
||||
- skipped unselected outputs
|
||||
- locked outputs
|
||||
- one `publish_output_outcomes` collection containing all five output states,
|
||||
exact declared keys, and key-derived origin runs where applicable
|
||||
- remote commit and current-pointer key paths
|
||||
- the run identifier selected by the commit
|
||||
|
||||
@@ -83,35 +104,44 @@ Includes counts/lists for:
|
||||
from verified descriptors; symlinks, special files, replacement races, and
|
||||
undeclared entries are rejected or ignored before uploads begin.
|
||||
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||
archived only when recorded by the run manifest.
|
||||
- publish locks are not overridden by `--force`; remote locks are revalidated
|
||||
immediately before current-state selection.
|
||||
- post-commit local cleanup is authorized by the committed publish metadata and
|
||||
is durably recorded by the application lifecycle before any local deletion.
|
||||
archived only when recorded by the session manifest. A run with no declared
|
||||
current-run diagnostics legitimately uploads none.
|
||||
- independent publish locks are not overridden by `--force`; every initially
|
||||
unlocked relevant source is reread immediately before current-state
|
||||
selection.
|
||||
- successful uploaded publication records the exact run ID, remote commit key,
|
||||
and current-pointer key consumed by the later cleanup stage.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
|
||||
normalized source/destination/required output rules, static lock policy, and
|
||||
the remote backend, bucket, region, endpoint, and root-prefix identity. Rule
|
||||
and lock ordering is canonicalized. Credential environment names,
|
||||
path-addressing transport mode, local workspace placement, and run identifiers
|
||||
are excluded. Remote locks remain mutable state and are still revalidated at
|
||||
the commit boundary; semantic evidence does not replace that safety check.
|
||||
normalized source/destination/required output rules, and the remote backend,
|
||||
bucket, region, endpoint, and root-prefix identity. Rule ordering is
|
||||
canonicalized. Credential
|
||||
environment names, path-addressing transport mode, local workspace placement,
|
||||
and run identifiers are excluded. Independent remote locks remain mutable state
|
||||
and are read directly by publish at planning and commit boundaries; semantic
|
||||
evidence does not replace those safety checks.
|
||||
|
||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||
The commit boundary and cleanup authorization are normative architecture invariants; see
|
||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||
|
||||
Publish never deletes local state. The standalone `publish` command selects
|
||||
only this stage; a full run selects notify and cleanup afterward.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output and
|
||||
static-lock fields.
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output
|
||||
rules.
|
||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||
physical remote state.
|
||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||
helpers.
|
||||
- Implementation and tests: `internal/stage/publish.go`,
|
||||
`internal/stage/publish_sources.go`, `internal/stage/publish_outputs.go`,
|
||||
`internal/stage/publish_output_plan.go`,
|
||||
`internal/stage/publish_transfer.go`, `internal/stage/publish_commit.go`,
|
||||
`internal/stage/publish_test.go`,
|
||||
`internal/stage/semantic_contracts_delivery.go`,
|
||||
`internal/app/operator_helpers_test.go`, and
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
|
||||
@@ -21,7 +21,7 @@ records that may consume rendered transcripts.
|
||||
## Key Behavior
|
||||
|
||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||
- resolves inputs manifest-first, then canonical fallback.
|
||||
- resolves inputs only from verified manifest output evidence.
|
||||
- writes run-local outputs first, then materializes canonical session outputs.
|
||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||
|
||||
@@ -16,7 +16,7 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||
- derives the transcript identity from the prepared `.flac` filename.
|
||||
- dispatches WhisperX requests through a bounded worker pool.
|
||||
- dispatches one WhisperX request per prepared file through a bounded errgroup.
|
||||
- validates each output as JSON.
|
||||
- writes run-local outputs then materializes canonical transcript outputs only
|
||||
after every planned request succeeds.
|
||||
|
||||
@@ -16,6 +16,7 @@ operator-selected storage fields and credential mechanisms belong in
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||
- `Delete(ctx, key)` removes one object and succeeds when it is already absent
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
@@ -47,6 +48,8 @@ not own discovery, defaults, or configuration validation.
|
||||
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||
match or absence precondition directly to the provider request and reports a
|
||||
failed precondition without performing a local check-then-write replacement.
|
||||
- `Delete` maps one normalized key to the provider delete-object operation and
|
||||
treats an absent object as success.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Invariants
|
||||
@@ -55,6 +58,11 @@ not own discovery, defaults, or configuration validation.
|
||||
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||
not replace owner-specific size policy.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
- A caller that installs remote content into a canonical workspace path first
|
||||
writes a sibling temporary file, validates the complete object against its
|
||||
owner-specific contract, and only then replaces the destination. Temporary
|
||||
files used solely for remote comparisons are removed by their callers and do
|
||||
not represent installed workspace state.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@ path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||
consume those helpers instead of rebuilding relative paths.
|
||||
|
||||
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||
cleanup roots and their descendants through no-follow directory handles before
|
||||
removing them.
|
||||
explicit cleanup roots and removes only normalized proper descendants through
|
||||
the resulting root handle.
|
||||
|
||||
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||
@@ -33,15 +33,16 @@ previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||
copies only regular files and directories to a same-filesystem temporary
|
||||
sibling. Source traversal uses confined directory handles and identity checks
|
||||
so replacing an inspected root, directory, or file is rejected rather than
|
||||
followed. The completed tree is atomically renamed without replacing an
|
||||
existing destination. Exact physical paths belong in
|
||||
validates and renames the external bundle to the unique immutable Notarius
|
||||
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
Prepared references delivered to Notarius use session-durable paths owned by
|
||||
`internal/artifacts` beneath
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
|
||||
content-addressed snapshots are outside run-local layout, survive ordinary
|
||||
published-run cleanup, and are excluded from publish and restore enumeration.
|
||||
|
||||
## Locking
|
||||
|
||||
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||
@@ -50,21 +51,25 @@ file retains owner metadata after release or process death; its existence is
|
||||
not evidence that a lock is active. Command and restore flows wait for this
|
||||
lock only while their context remains active, and report a release failure.
|
||||
|
||||
The runner assigns every invocation a run-workspace identity but creates the
|
||||
physical directory only before an executing stage that requires it. Fully
|
||||
reused invocations and cleanup-only invocations therefore do not accumulate
|
||||
empty run directories.
|
||||
|
||||
## Cleanup Semantics
|
||||
|
||||
Automatic post-publish cleanup:
|
||||
The canonical cleanup stage:
|
||||
|
||||
- is created only after a successful publish commit with complete publish
|
||||
metadata, then is persisted before any deletion;
|
||||
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
|
||||
key in publish metadata;
|
||||
- runs after notify only when selected by the invocation range;
|
||||
- requires a succeeded publish record with `uploaded=true`, the published run
|
||||
ID, remote commit key, and current commit-pointer key;
|
||||
- consumes the resolved cleanup policy described in
|
||||
[Configuration](../config.md);
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
|
||||
ancestors or entries);
|
||||
- retries any recorded incomplete target on later invocations even when no
|
||||
publish work is selected. Missing targets are a successful, idempotent
|
||||
cleanup result only after the completion evidence is saved.
|
||||
- refuses unsafe deletes (root delete, traversal, and out-of-root targets);
|
||||
- treats missing targets as successful, idempotent deletion; and
|
||||
- uses its ordinary failed/succeeded manifest stage record for retry. A later
|
||||
full run can skip reusable upstream stages and retry cleanup without
|
||||
republishing.
|
||||
|
||||
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
@@ -75,6 +80,8 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
- restore installation is confined to validated relative paths beneath the
|
||||
selected session root.
|
||||
- ordinary workspace paths retain group-writable directory and file modes across
|
||||
nested creation, replacement, and Notarius promotion.
|
||||
|
||||
@@ -83,12 +90,14 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||
- Immutable bundle installation: `internal/stage/extract.go`
|
||||
- Workspace modes: `internal/fileops/modes.go`
|
||||
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
|
||||
`internal/stage/cleanup.go`, `internal/stage/cleanup_plan.go`, and
|
||||
`internal/app/cleanup_targets.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
|
||||
@@ -76,12 +76,15 @@ value from the last value persisted in the session manifest, which helps
|
||||
diagnose profile switches without changing resume authority.
|
||||
|
||||
Before switching an operational profile, compare its effective meaning with the
|
||||
current selection through `narratio config diff <left-profile> <right-profile>`.
|
||||
production base through `narratio config diff default <profile>`.
|
||||
The command is read-only and succeeds whether it finds differences or not. Its
|
||||
sorted records describe defaulted, expanded concrete configuration—not source
|
||||
file layout—so it can be used to review model, artifact, and publish changes
|
||||
without creating a session or run. Select the same campaign explicitly when
|
||||
profiles could resolve different campaign paths; see the [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) for syntax and record format.
|
||||
without creating a session or run. Use `narratio config artifacts` to review the
|
||||
concrete artifact identities, definitions, outputs, and dependencies. Select the
|
||||
same campaign explicitly when profiles could resolve different campaign paths;
|
||||
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
for syntax and record format.
|
||||
|
||||
## Stage Execution and Continuation Behavior
|
||||
|
||||
@@ -98,12 +101,12 @@ Canonical stage order:
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
12. `cleanup`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set; stages with semantic
|
||||
configuration contracts additionally require matching versioned evidence,
|
||||
and missing legacy evidence causes a safe one-time rerun;
|
||||
configuration contracts additionally require matching versioned evidence;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- forcing a stage marks succeeded transitive dependents as `stale` before the
|
||||
replacement runs; render and extract are independent siblings; and
|
||||
@@ -155,7 +158,7 @@ narratio run 2026-04-04 --from extract --through analyze --force
|
||||
```
|
||||
|
||||
Omitting `--from` selects from `prepare`; omitting `--through` selects through
|
||||
`notify`. Force applies only within the selected range. Repeating `--from`,
|
||||
`cleanup`. Force applies only within the selected range. Repeating `--from`,
|
||||
`--through`, or `--force` is rejected instead of resolving by argument order.
|
||||
The plan command uses the same selection contract and prints only the selected
|
||||
range. Planning is read-only: it clones the loaded manifest, models selected
|
||||
@@ -169,11 +172,11 @@ selected artifact evidence is still semantically current.
|
||||
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
||||
prefix stage must already have a session-manifest status of `succeeded` or
|
||||
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
||||
or interrupted prerequisite without creating a run record or changing session
|
||||
state. Widen `--from` to include that stage, or recover it explicitly before
|
||||
retrying. Excluded prefix stages are not resume-validated or repaired as part
|
||||
of the bounded invocation; selected stages still reject missing, unsafe, or
|
||||
manifest-inconsistent inputs at their owning boundary.
|
||||
or interrupted prerequisite without creating a run-local directory or changing
|
||||
session state. Widen `--from` to include that stage, or recover it explicitly
|
||||
before retrying. Excluded prefix stages are not resume-validated or repaired as
|
||||
part of the bounded invocation; selected stages still reject missing, unsafe,
|
||||
or manifest-inconsistent inputs at their owning boundary.
|
||||
|
||||
Stages after `--through` are not prerequisites and are never scheduled by the
|
||||
bounded invocation. A selected forced stage can mark one of those succeeded
|
||||
@@ -184,7 +187,20 @@ shared session lifecycle. In particular, render does not require Notarius or
|
||||
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
||||
the transcription, Seriatim, Audita, or Notarius adapters.
|
||||
|
||||
For the common post-transcript development loop, use:
|
||||
## Artifact Development Workflows
|
||||
|
||||
Choose the narrowest workflow that matches the intended replacement:
|
||||
|
||||
| Intent | Command | Preserved and rebuilt state |
|
||||
| --- | --- | --- |
|
||||
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
|
||||
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
|
||||
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
|
||||
|
||||
All three stop before publication. Inspect the results and run
|
||||
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
|
||||
|
||||
For the extraction-plus-analysis development loop, use:
|
||||
|
||||
```bash
|
||||
narratio regenerate-artifacts 2026-04-04
|
||||
@@ -195,8 +211,8 @@ This command is a transparent expansion to a forced bounded `run` from
|
||||
`extract` through `analyze`. Extraction always rebuilds its complete configured
|
||||
bundle. Analysis rebuilds the selected targets and their required analysis
|
||||
prerequisites, or uses the normal default selection when no artifact names are
|
||||
given. The command does not run publish or notify; delivery remains a separate
|
||||
operator action.
|
||||
given. The command does not run publish, notify, or cleanup; delivery and
|
||||
retention remain separate operator actions.
|
||||
|
||||
Inspect current artifact evidence, then publish explicitly when the regenerated
|
||||
set is ready:
|
||||
@@ -206,11 +222,17 @@ narratio session artifacts 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
If planning or execution reports stale, missing, failed, legacy, or tampered
|
||||
If planning or execution reports stale, missing, failed, or tampered
|
||||
analysis evidence, regenerate the affected target instead of copying an older
|
||||
canonical file into place or editing the manifest. See
|
||||
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
|
||||
|
||||
Analyze checkpoints each completed configured artifact before starting later
|
||||
artifact work. After a partial failure or interruption, retry normally: valid
|
||||
earlier checkpoints are reused and execution resumes with failed, stale, or
|
||||
missing work in dependency order. `--force` is unnecessary unless the operator
|
||||
intends to replace otherwise-current artifacts.
|
||||
|
||||
## Artifact Selection
|
||||
|
||||
For a configured artifact family, selecting its family key expands to every
|
||||
@@ -251,9 +273,14 @@ are not selectable or published implicitly.
|
||||
Configured Notarius references resolve only from the current manifest-backed
|
||||
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
||||
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
||||
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified copies
|
||||
under `runs/<run_id>/extract/references/` so a concurrent refresh of canonical
|
||||
prepared files cannot change the bytes consumed by an in-flight invocation.
|
||||
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
|
||||
content-addressed copies under
|
||||
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
|
||||
SHA-256 digest identifies the reference bytes, so retries and later invocations
|
||||
reuse the same path while changed bytes select a new path. Older snapshots are
|
||||
retained and are not current artifact evidence. A concurrent refresh of
|
||||
canonical prepared files therefore cannot change the bytes consumed by an
|
||||
in-flight invocation.
|
||||
For a canonical party, preparation retains the validated authored party bytes
|
||||
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
||||
The manifest records their checksums separately, with the players input marked
|
||||
@@ -280,15 +307,35 @@ narratio run-stage prepare 2026-04-04 --force
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||
immutable bundle remain available for inspection, but downstream resolution
|
||||
requires a new current successful extraction record.
|
||||
record does not fall back to the earlier outputs. The earlier immutable bundle
|
||||
remains available for inspection, but downstream resolution requires a new
|
||||
current successful extraction record.
|
||||
|
||||
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
|
||||
and other operating systems, extraction fails before copying the bundle into a
|
||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||
directory primitive there. This is an extraction limitation, not a broader
|
||||
platform-support guarantee for every Narratio workflow.
|
||||
Notarius extraction validates its run-local bundle and moves it once to the
|
||||
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||
no duplicate promotion tree is created.
|
||||
|
||||
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
|
||||
inside an executed Notarius invocation. Ordinary extraction forwards their
|
||||
configured values. Forced extraction always disables checkpoint loading and
|
||||
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
|
||||
Refresh regenerates and replaces only the active input's accepted chunk plan;
|
||||
it is not cache deletion.
|
||||
|
||||
Notarius checkpoint recording must be enabled in the deployed Notarius
|
||||
configuration. Narratio neither creates a Notarius pipeline/configuration nor
|
||||
reads it to infer checkpoint availability. Configure explicit persistent
|
||||
Notarius chunk-plan and checkpoint roots when reuse must survive separate
|
||||
Narratio subprocesses. A changed reference, transcript, selected profile,
|
||||
profile content, endpoint, or other Notarius checkpoint identity input can
|
||||
produce a legitimate cold miss.
|
||||
|
||||
There are two independent resume boundaries: Narratio may skip a completed
|
||||
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
|
||||
reuse compatible accepted work only after Narratio has decided extraction must
|
||||
execute. Deleting Narratio's local reference snapshots does not invalidate a
|
||||
complete durable extraction; missing snapshots are reconstructed only when a
|
||||
later extraction actually runs.
|
||||
|
||||
## External Command Lifecycle
|
||||
|
||||
@@ -311,12 +358,15 @@ Run-local diagnostics are:
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run-record upload is an allowlist derived from the validated run manifest,
|
||||
not a workspace scan. Each declared source is opened without following
|
||||
symlinked ancestors or the leaf, verified as a regular file, and streamed from
|
||||
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
|
||||
The durable bundle is never scanned for implicit publication; only lanes named
|
||||
by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
The run upload is an allowlist derived from the session manifest's declared
|
||||
current-run outputs, logs, and generated configurations, not a workspace scan.
|
||||
References to canonical session files and prior-run paths are excluded. Each
|
||||
declared source is opened without following symlinked ancestors or the leaf,
|
||||
verified as a regular file, and streamed from that verified descriptor.
|
||||
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
||||
current-run diagnostics uploads no run-local files. No distinguished diagnostic
|
||||
file is uploaded. The durable bundle is never scanned for implicit publication;
|
||||
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
@@ -339,10 +389,11 @@ outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
Publish reuse additionally tracks enabled/run-upload behavior, normalized
|
||||
output rules, static locks, and remote backend/bucket/region/endpoint/root
|
||||
identity. Credential environment names, local workspace placement, and run IDs
|
||||
are excluded. Regardless of semantic reuse evidence, executing publish still
|
||||
revalidates mutable remote locks immediately before commit selection.
|
||||
output rules, and remote backend/bucket/region/endpoint/root identity.
|
||||
Credential environment names, local workspace placement, and run IDs are
|
||||
excluded. Regardless of semantic reuse evidence, executing publish reads
|
||||
independent remote locks directly and rechecks initially unlocked sources
|
||||
before commit selection.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
@@ -358,6 +409,9 @@ Equivalent:
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
This standalone command stops after publish and does not invoke cleanup. A
|
||||
default full `run` continues through notify and cleanup.
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
@@ -381,31 +435,44 @@ commit by version, checksum, size, and storage generation; committed artifacts
|
||||
are also checksum- and generation-bound. Readers accept this contract now and
|
||||
strictly reject mismatched or unknown data.
|
||||
|
||||
Legacy reads are limited to a coherent `current/manifest.json` and
|
||||
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
|
||||
write that pair and remote commit state does not carry local
|
||||
`current_pointer_written` metadata.
|
||||
A selected commit may carry a `published_output` declaration from an earlier
|
||||
run in the same campaign/session. Restore, previous-session caching, status,
|
||||
and artifact inspection use that declaration's exact immutable key and stable
|
||||
source; they do not reconstruct a current-run key or scan the session prefix.
|
||||
Other artifact types remain confined to the selected commit's own run.
|
||||
|
||||
When the pointer is absent, remote state is not published; republish the
|
||||
session to establish a committed snapshot. Remote commit state does not carry
|
||||
local `current_pointer_written` metadata.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
Lock sources:
|
||||
Each mutable lock is an independent strict JSON object at
|
||||
`{session_prefix}/locks/{sha256(source)}.lock`. The lowercase digest is computed
|
||||
from the exact canonical source. Listing validates every object in that
|
||||
reserved prefix and reports locks sorted by source.
|
||||
|
||||
- static locks in `pipeline.publish.locks`
|
||||
- mutable remote locks in `{session_prefix}/locks.yml`
|
||||
Upgrades do not read, migrate, or delete the retired `{session_prefix}/locks.yml`
|
||||
object. It has no effect on publication and may be removed manually after
|
||||
verifying lock commands and publication behavior with the upgraded Narratio
|
||||
version. No migration command is required.
|
||||
|
||||
Effective lock rules:
|
||||
Operational rules:
|
||||
|
||||
- static and remote locks are merged;
|
||||
- static locks win on source collisions;
|
||||
- locked outputs are intentional skips;
|
||||
- lock add/remove commands mutate only remote lock state through generation-bound
|
||||
conditional writes. A command retries a bounded number of concurrent
|
||||
conflicts while its invocation context remains active, so it never replaces a
|
||||
different lock-document generation; and
|
||||
- a publish re-reads remote locks immediately before it writes the current
|
||||
commit pointer. A lock committed before that recheck prevents selecting the
|
||||
new snapshot, even though its already-uploaded immutable objects may remain
|
||||
available for a later retry.
|
||||
- add uses conditional creation and reports an existing lock;
|
||||
- add with `--force` observes the current generation and attempts one
|
||||
generation-matched replacement. A concurrent change is reported for an
|
||||
explicit operator retry;
|
||||
- remove is idempotent and reports whether a lock was present;
|
||||
- the initial publish lock snapshot controls output planning. A locked source
|
||||
carries its exact declaration from the prior current commit when available;
|
||||
a never-published locked source remains absent even when required;
|
||||
- carried bytes are not copied. Publish verifies the prior object's exact key,
|
||||
size, and generation without downloading it; and
|
||||
- after immutable uploads and the immutable commit, publish re-reads every
|
||||
initially unlocked participating source. A newly added lock prevents the
|
||||
current pointer from advancing, although the immutable attempt may remain.
|
||||
A removed initial lock may conservatively cause one carry.
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -439,13 +506,22 @@ Default restore scope:
|
||||
|
||||
- the committed session manifest and the committed transcript/artifact objects
|
||||
declared by the selected remote commit
|
||||
- `previous/**` when needed by configured previous-session artifact inputs
|
||||
- `previous/**` when needed by configured previous-session artifact inputs;
|
||||
each is selected from its exact committed `published_output` declaration
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||
directories or other non-regular local targets
|
||||
directories, symlinks, or other non-regular local targets
|
||||
|
||||
Restore treats a local leaf symlink or any symlinked ancestor beneath the
|
||||
selected session root as a conflict, even when the linked content matches the
|
||||
selected remote object. `--force` does not override that conflict.
|
||||
|
||||
Before each install, restore validates the destination again beneath the
|
||||
selected session root. A target changed into a symlink, directory, or other
|
||||
non-regular entry is not overwritten.
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
@@ -473,10 +549,24 @@ Durable session paths:
|
||||
- `logs/**`
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
- `cache/notarius/references/<sha256>/<prepared-basename>`
|
||||
|
||||
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||
add missing markers. Move it aside and regenerate it, restore a verified
|
||||
current committed snapshot, or republish healthy regenerated state as the
|
||||
relevant workflow requires.
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
Stable reference snapshots live in the session cache path above. Ordinary
|
||||
published-run cleanup removes its selected run-local target but retains this
|
||||
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
|
||||
or digest directory manually only while no command is active for the session;
|
||||
regular missing or corrupt content is reconstructed on the next executed
|
||||
extraction, while symlinks, directories, and other unsafe conflicts must be
|
||||
investigated and removed deliberately.
|
||||
|
||||
Run-local layout:
|
||||
|
||||
@@ -498,6 +588,8 @@ Cache layout (durable S3 audio cache):
|
||||
Each cached audio file has an adjacent managed identity record. It binds the
|
||||
file to its remote object version and verified digest; deleting or altering the
|
||||
record simply causes Narratio to download and verify the object again.
|
||||
Remote audio and restored files are fully transferred into sibling temporary
|
||||
files and validated before they replace their canonical workspace paths.
|
||||
|
||||
### Workspace Permissions
|
||||
|
||||
@@ -509,18 +601,28 @@ converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
||||
caller umask does not remove group write or setgid. It does not change file or
|
||||
directory ownership: the configured workspace's existing group is inherited.
|
||||
|
||||
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
||||
workspace, spool, and cache locations with an ACL that grants the collaborating
|
||||
group read/write access, and configure credential locations with an ACL limited
|
||||
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
||||
Windows access control.
|
||||
|
||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||
shared workspace or in a separately restricted credential location; ordinary
|
||||
workspace group access must never be treated as authorization to read keys.
|
||||
On POSIX, provision a credential directory as `0700` and credential files as
|
||||
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
||||
On Windows, restrict the directory and files with ACLs to the credential owner.
|
||||
Provision a credential directory as `0700` and credential files as `0600`;
|
||||
these permission modes are an operator requirement and are not enforced by
|
||||
ConfigKit.
|
||||
|
||||
Every pipeline-loading command bootstraps credentials once, before the first
|
||||
configuration load and before any adapter is constructed. `--secrets-dir`
|
||||
selects an explicit directory (relative paths use caller CWD). Otherwise a
|
||||
present `secrets/` beside the resolved root pipeline is used, while an absent
|
||||
conventional directory is skipped. Explicit missing/empty selections and
|
||||
invalid existing directories fail the command. ConfigKit follows a symlink used
|
||||
as the selected directory, but immediate entries must be regular non-symlink
|
||||
files. Existing environment variables—including empty values—always win.
|
||||
|
||||
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
|
||||
environment references. When both are absent, the AWS SDK retains its default
|
||||
credential provider chain. When both are configured, Narratio supplies their
|
||||
load-time resolved values through an explicit static provider. Audita's optional
|
||||
`llm_api_key` is likewise captured at pipeline load and supplied only through
|
||||
the child process's `AUDITA_LLM_API_KEY` override.
|
||||
|
||||
External adapter results are individually bounded before Narratio validates or
|
||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||
@@ -553,17 +655,29 @@ Rules:
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||
root and refuses symlinked paths;
|
||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||
root; root, traversal, and out-of-root targets are refused;
|
||||
- the final cleanup stage is gated by successful publish evidence plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
|
||||
reports incomplete, the remote committed snapshot remains current; rerun
|
||||
publish to retry only the outstanding confined local cleanup.
|
||||
- the manifest publish record has `uploaded=true`, the published run ID, the
|
||||
exact remote commit key, and the current commit-pointer key.
|
||||
|
||||
Post-publish cleanup is evaluated only when `publish` actually executes in the
|
||||
current invocation. A bounded range that excludes publish does not replay a
|
||||
cleanup obligation as an unrelated side effect.
|
||||
Cleanup derives only the configured run-scoped spool and workspace targets,
|
||||
uses root-confined deletion, and does not read or mutate object storage. A
|
||||
failure leaves publish succeeded and the remote committed snapshot current.
|
||||
Retry with a normal full run: valid upstream work, including publish and notify,
|
||||
is skipped while the failed cleanup stage runs again. `run-stage cleanup` is
|
||||
also available when the manifest has valid publish evidence. A range ending at
|
||||
`notify` or earlier and the standalone `publish` command never delete local
|
||||
state.
|
||||
|
||||
Older local manifests that contain the retired cleanup-obligation field are
|
||||
intentionally incompatible with strict manifest decoding. If such an
|
||||
obligation is still outstanding, complete it with the pre-upgrade binary.
|
||||
Otherwise verify that the remote publication is committed, move aside or
|
||||
regenerate the incompatible local manifest (or restore the verified remote
|
||||
snapshot), and then use the cleanup stage. Manual JSON editing is not the
|
||||
normal migration path.
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
@@ -571,4 +685,5 @@ cleanup obligation as an unrelated side effect.
|
||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||
- Restore requires configured object storage and committed remote current state.
|
||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
||||
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
|
||||
resolution and object-store initialization.
|
||||
|
||||
@@ -116,8 +116,11 @@ adapter and publish metadata.
|
||||
|
||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||
inspected and validated, but file presence alone does not replace recorded
|
||||
stage state. Invocation-scoped run records provide an audit of one execution;
|
||||
they do not replace the session manifest as progress authority.
|
||||
stage state. Each invocation retains an in-memory command summary and a
|
||||
run-local workspace identity, but neither is a durable progress ledger. The
|
||||
physical run directory is created on demand only when an executing stage
|
||||
requires it; a fully reused invocation or a cleanup-only invocation does not
|
||||
leave an empty run directory.
|
||||
|
||||
Restore treats committed remote current state as its authority. It must plan
|
||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||
@@ -143,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
|
||||
- YAML decoding rejects unknown fields.
|
||||
- Defaults are centralized and testable.
|
||||
- Empty configured values do not silently replace meaningful defaults.
|
||||
- Validation rejects invalid composition before stage execution where
|
||||
- Validation rejects invalid configuration before stage execution where
|
||||
practical.
|
||||
- Root-owned imports and one selected profile resolve deterministically through
|
||||
the configuration owner; commands do not implement their own merge rules.
|
||||
@@ -185,8 +188,8 @@ availability.
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
relative destinations must not expand the scope of an operation. Cleanup is
|
||||
permitted only through explicit operator action or configured post-publish
|
||||
gates, and it must preserve durable cache unless cache removal is explicitly
|
||||
permitted only through explicit operator action or the configured final cleanup
|
||||
stage, and it must preserve durable cache unless cache removal is explicitly
|
||||
requested.
|
||||
|
||||
Physical layout, retention, and operational lifecycle belong in
|
||||
@@ -202,16 +205,26 @@ the immutable commit manifest, and finally the current commit pointer.
|
||||
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||
must not be presented as current remote state. Publish locks remain authoritative
|
||||
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
|
||||
generation preconditions and are revalidated immediately before pointer
|
||||
selection; loss of that check leaves the prior committed snapshot current.
|
||||
and are not bypassed by a forced run. Each mutable remote lock is an independent
|
||||
per-source object; lock mutations use provider-enforced generation preconditions.
|
||||
Publish revalidates every initially unlocked participating source after writing
|
||||
the immutable commit and before pointer selection. A lock observed then leaves
|
||||
the prior committed snapshot current. A lock created after that final check may
|
||||
race the pointer write; the commit protocol does not claim a stronger distributed
|
||||
transaction boundary.
|
||||
|
||||
Automatic local cleanup is permitted only after a successful publish commit,
|
||||
only when explicitly configured, and only through the path-safety guardrails.
|
||||
It is a durable local obligation bound to that committed run and its exact
|
||||
targets, not an inferred side effect of the current stage list. A cleanup
|
||||
failure makes the invocation incomplete while leaving the committed remote
|
||||
snapshot authoritative; later invocations resume the recorded obligation.
|
||||
It is the final ordinary pipeline stage after `notify`. The stage derives its
|
||||
exact run-scoped targets from validated publish metadata and the configured
|
||||
campaign/session identity; it never infers a target from directory contents or
|
||||
rereads remote storage. A cleanup failure makes the invocation incomplete while
|
||||
leaving publish succeeded and the committed remote snapshot authoritative. A
|
||||
normal later full run reuses valid upstream stages and retries cleanup.
|
||||
|
||||
The standalone `publish` command selects only publish and therefore never
|
||||
invokes cleanup. Operators may select cleanup directly only when the manifest
|
||||
contains valid successful publication evidence.
|
||||
|
||||
## Security, Privacy, And Diagnostics
|
||||
|
||||
@@ -221,20 +234,26 @@ configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
||||
intentionally shareable with the configured workspace group. API-key material
|
||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||
|
||||
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
||||
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
|
||||
a cross-build gate and release asset, but no native runtime release gate.
|
||||
Windows and macOS amd64 are unsupported and have no compatibility promise or
|
||||
release asset.
|
||||
|
||||
On supported systems, Narratio-created ordinary workspace directories converge on
|
||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||
umask is restrictive. This preserves the existing workspace group for nested
|
||||
creation and atomic replacements without changing ownership. API-key storage
|
||||
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
||||
are not authoritative; operators must provide the equivalent shared-group and
|
||||
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
||||
uses a separate restrictive contract.
|
||||
|
||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||
to manifests, logs, generated configuration, reports, publish metadata,
|
||||
documentation, or examples. Secrets enter through configured environment
|
||||
variable names or secret-file references. Diagnostics should avoid transcript
|
||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
||||
it.
|
||||
documentation, or examples. Pipeline configuration names environment variables;
|
||||
command-owned startup may populate previously unset variables from ConfigKit's
|
||||
explicit or conventional secrets directory. This bootstrap occurs once per
|
||||
invocation before any typed configuration load or adapter construction and is
|
||||
not hidden inside library loaders. Safe bootstrap diagnostics may identify
|
||||
files and statuses but never values. Diagnostics should avoid transcript and
|
||||
prompt content unless a deliberate, bounded inspection mechanism requires it.
|
||||
|
||||
Logs, reports, generated invocation files, generated configuration, and render
|
||||
debug files are diagnostics, not canonical pipeline products. They should be
|
||||
|
||||
@@ -5,6 +5,13 @@ binary release. The synchronous release boundary is a successful push of one
|
||||
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
|
||||
change that result.
|
||||
|
||||
## Go prerequisite
|
||||
|
||||
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
|
||||
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
|
||||
asset construction; that asynchronous work does not change the tag-push
|
||||
completion boundary.
|
||||
|
||||
## Choose a version and write its note
|
||||
|
||||
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
|
||||
@@ -51,14 +58,14 @@ scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
||||
|
||||
It validates the version and matching note, module hygiene, formatting,
|
||||
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
||||
and six official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||
and three official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||
application services, CI, or Gitea, and does not create tags or modify tracked
|
||||
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
||||
checker.
|
||||
|
||||
The checker builds Linux, macOS, and Windows assets for `amd64` and `arm64`.
|
||||
Cross-builds prove compilation; they are not native macOS or Windows runtime
|
||||
evidence.
|
||||
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
|
||||
`arm64`. The macOS cross-build proves compilation; it is not native macOS
|
||||
runtime evidence.
|
||||
|
||||
## Publish the tag
|
||||
|
||||
|
||||
@@ -22,5 +22,7 @@ See the [release procedure](../release.md) for creating a candidate and tag.
|
||||
When asynchronous publication succeeds, the corresponding Gitea release is the
|
||||
canonical source for downloadable binaries and checksums.
|
||||
|
||||
- [v2.0.0](v2.0.0.md)
|
||||
- [v1.7.0](v1.7.0.md)
|
||||
- [v1.6.0](v1.6.0.md)
|
||||
- [v1.5.0](v1.5.0.md)
|
||||
|
||||
100
docs/releases/v1.7.0.md
Normal file
100
docs/releases/v1.7.0.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Narratio v1.7.0
|
||||
|
||||
Narratio v1.7.0 completes the maintainability refactor begun after v1.6.0. It
|
||||
simplifies filesystem and process handling, makes resumable work more precise,
|
||||
replaces legacy local and remote state with explicit authorities, and turns
|
||||
cleanup into the final ordinary pipeline stage.
|
||||
|
||||
## Summary
|
||||
|
||||
The session manifest is now the sole durable local progress ledger. Analyze
|
||||
checkpoints successful artifacts individually, allowing an interrupted or
|
||||
partially failed analysis to resume at the first incomplete artifact without
|
||||
rerunning successful siblings unless forced.
|
||||
|
||||
Remote publication now commits immutable run state before advancing one
|
||||
verified current pointer. Publish locks are independent per-source objects;
|
||||
locked outputs retain their previously committed declarations rather than
|
||||
being overwritten or silently omitted. Local cleanup runs after notification
|
||||
as a separately recorded, safely retryable stage.
|
||||
|
||||
The release also adopts Go 1.26, simplifies ordinary file operations around
|
||||
Narratio's trusted group-owned workspace model, improves subprocess lifecycle
|
||||
handling, decomposes several large orchestration and validation components,
|
||||
and removes mutable global test seams.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a minor release with deliberate removal of compatibility paths that
|
||||
were deprecated and documented as subject to removal:
|
||||
|
||||
- Campaigns must use one versioned `narratio.party.v1` `party.yml`. Separate
|
||||
players files, session-level party overrides, and unversioned party documents
|
||||
are rejected.
|
||||
- Session manifests must declare `schema_version:
|
||||
narratio.session-manifest.v1` and satisfy the current strict schema. Older
|
||||
unversioned state and manifests containing the retired
|
||||
`post_publish_cleanup` obligation are rejected rather than migrated.
|
||||
- Remote current state must be selected by
|
||||
`current/commit-pointer.json` and its referenced immutable run commit. Legacy
|
||||
current-state layouts are no longer inferred.
|
||||
- The retired aggregate `locks.yml` object is ignored. Publish locks now live
|
||||
as independent strict objects under the session lock prefix.
|
||||
- Run manifests are no longer produced or consumed. The session manifest owns
|
||||
durable stage progress; invocation summaries and run workspaces remain
|
||||
operational data rather than a second progress authority.
|
||||
|
||||
The CLI and current pipeline/configuration formats otherwise remain compatible.
|
||||
Building and maintaining Narratio now requires Go 1.26. Cross-built Linux,
|
||||
macOS, and Windows assets remain available for `amd64` and `arm64`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Before upgrading a campaign that still uses compatibility data:
|
||||
|
||||
1. Replace legacy party/player inputs with the documented
|
||||
[`narratio.party.v1`](../integrations/party.md) campaign roster.
|
||||
2. If a local manifest contains an outstanding `post_publish_cleanup`
|
||||
obligation, complete it with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, or
|
||||
restore a verified committed snapshot. Do not migrate state by manually
|
||||
editing JSON.
|
||||
3. Republish sessions that do not yet have a valid immutable run commit and
|
||||
`current/commit-pointer.json`; legacy remote layouts are not current state in
|
||||
v1.7.0.
|
||||
4. Recreate any required publication locks with `narratio session locks add`.
|
||||
After verifying the new locks and publication behavior, the ignored legacy
|
||||
`locks.yml` object may be removed manually.
|
||||
|
||||
Existing compatible session manifests and already committed remote snapshots
|
||||
need no special action. Consult [Operations](../operations.md) and
|
||||
[Troubleshooting](../troubleshooting.md) for recovery procedures.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded the maintained development and release baseline to Go 1.26.
|
||||
- Reworked subprocess execution and Windows process-tree supervision while
|
||||
preserving contextual wait and cleanup failures.
|
||||
- Replaced custom confined ordinary I/O and directory-promotion machinery with
|
||||
simpler standard-library operations while retaining bounded reads, atomic
|
||||
single-file visibility, workspace modes, validated downloads, and
|
||||
root-confined destructive operations.
|
||||
- Removed legacy party, manifest, artifact-discovery, remote-current, and lock
|
||||
compatibility paths.
|
||||
- Consolidated durable progress in the session manifest and removed the run
|
||||
manifest domain model.
|
||||
- Unified runner planning and execution decisions, separated invocation
|
||||
lifecycle responsibilities, and avoided creating empty run workspaces for
|
||||
fully reused or cleanup-only invocations.
|
||||
- Added per-artifact analyze execution, checkpointing, reconciliation, and
|
||||
recovery.
|
||||
- Decomposed configuration validation by domain and made validation diagnostics
|
||||
deterministic without treating incidental ordering or wording as a contract.
|
||||
- Introduced immutable remote publish commits, one final current pointer,
|
||||
explicit output planning, safe cross-run carried outputs, and independent
|
||||
per-artifact remote locks.
|
||||
- Added `cleanup` as the final canonical stage after `notify`, with validated
|
||||
publish evidence, safe retry, explicit selection, and no separate cleanup
|
||||
obligation.
|
||||
- Replaced mutable command test hooks with invocation-scoped dependencies and
|
||||
improved structural test diagnostics.
|
||||
94
docs/releases/v2.0.0.md
Normal file
94
docs/releases/v2.0.0.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Narratio v2.0.0
|
||||
|
||||
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
|
||||
simplifies the supported release platforms, and makes artifact definitions and
|
||||
their effective dependency graph directly inspectable.
|
||||
|
||||
## Summary
|
||||
|
||||
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
|
||||
imports, fixed-path named profiles, provenance, defaults, and typed credential
|
||||
references. The recommended production layout keeps production values in the
|
||||
base, defines each ordinary artifact and artifact family in its own imported
|
||||
file, and keeps testing changes in `profiles/testing.yml`.
|
||||
|
||||
Configured-artifact inputs now create their dependency edges automatically.
|
||||
`narratio config artifacts` reports the finalized concrete catalog, including
|
||||
generated party-family members, without loading a session or initializing
|
||||
runtime adapters.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a breaking configuration and platform release. There is no runtime
|
||||
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
|
||||
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
|
||||
presence/default behavior. Defaults apply only to absent values; explicit empty,
|
||||
zero, false, and null values retain their typed meaning and may fail Narratio
|
||||
domain validation.
|
||||
|
||||
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
|
||||
and macOS amd64 builds are no longer produced or supported. Existing v1 release
|
||||
notes remain historical descriptions of their releases.
|
||||
|
||||
Inspection commands still avoid session, manifest, workspace, adapter, and
|
||||
remote-service activity, but they are not credential-free loaders: every
|
||||
explicit typed credential reference in the selected effective configuration must
|
||||
resolve; references replaced or removed by that profile need not resolve.
|
||||
Pipelines containing artifact families also require a campaign so the canonical
|
||||
party can be expanded.
|
||||
`config show` is a secret-safe review projection, not a promised reloadable
|
||||
authoring file.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Migrate each operational bundle before running v2:
|
||||
|
||||
1. Replace old composition syntax with root-owned
|
||||
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
|
||||
additive base sources; they do not override one another.
|
||||
2. Fold the old production overlay into the base pipeline. Move testing changes
|
||||
to `profiles/testing.yml`; compare the two independent resolutions with
|
||||
`narratio config diff default testing`.
|
||||
3. Follow the maintained
|
||||
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
|
||||
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
|
||||
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
|
||||
stable artifact identity.
|
||||
4. Move credential files to an untracked `secrets/` directory beside
|
||||
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
|
||||
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
|
||||
5. For static S3 credentials, configure both S3 references (commonly
|
||||
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
|
||||
To use instance roles, workload identity, shared SDK files, or another AWS SDK
|
||||
discovery provider, omit both references. A partial pair is invalid.
|
||||
6. Remove a `depends_on` entry when it only duplicates a
|
||||
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
|
||||
without supplying the upstream output as an input.
|
||||
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
|
||||
that previously acquired defaults after being explicitly empty. Do not use
|
||||
`config show` output as a source migration mechanism.
|
||||
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
|
||||
with the same profile, campaign, and secrets selection intended for runtime.
|
||||
Resolve the campaign-owned party whenever families are present.
|
||||
9. Update deployment and download automation to one of the three maintained
|
||||
platform assets before removing the v1 binary.
|
||||
|
||||
The [configuration reference](../config.md), [CLI reference](../cli.md), and
|
||||
[maintained examples](../../examples/README.md) are the canonical detailed
|
||||
guides.
|
||||
|
||||
## Changes
|
||||
|
||||
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
|
||||
while keeping Narratio domain validation and post-campaign expansion explicit.
|
||||
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
|
||||
unified safe effective show/diff/digest projection, and source provenance.
|
||||
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
|
||||
Audita credential references without exposing resolved values.
|
||||
- Inferred, normalized, and validated configured-artifact input dependencies as
|
||||
part of the one concrete graph consumed by analyze planning and resume logic.
|
||||
- Added `config artifacts` with stable concrete identities, definition sources,
|
||||
effective dependencies, and generated family/party provenance.
|
||||
- Added the maintained production-base/testing-overlay artifact authoring layout.
|
||||
- Reduced release assets and active platform support to Linux amd64/arm64 and
|
||||
macOS arm64.
|
||||
70
docs/releases/v2.1.0.md
Normal file
70
docs/releases/v2.1.0.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# Narratio v2.1.0
|
||||
|
||||
This release adds explicit Notarius credential forwarding and cache reuse,
|
||||
with predictable forced regeneration and clearer artifact-development guidance.
|
||||
|
||||
## Summary
|
||||
|
||||
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
|
||||
its accepted execution checkpoints. Stable, verified reference snapshots allow
|
||||
unchanged references to retain their cache identity across Narratio invocations.
|
||||
Configuration loading now uses ConfigKit v0.3.0.
|
||||
|
||||
## Compatibility
|
||||
|
||||
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
|
||||
fields are optional. Existing v2 configuration needs no schema migration.
|
||||
Configured credential references must resolve during configuration loading,
|
||||
even when extraction will not execute; omitted references add no requirement.
|
||||
|
||||
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
|
||||
refresh. This includes `regenerate-artifacts`, which remains an alias for a
|
||||
forced run from extract through analyze. `analyze` continues to force analysis
|
||||
only, preserving extraction results and transcripts.
|
||||
|
||||
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
|
||||
minimum-version gate. The executable must support the flags selected for the
|
||||
invocation, including `--chunk_cache refresh` for forced extraction. Reference
|
||||
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
|
||||
schemas are unchanged; reused producers may report zero production attempts.
|
||||
|
||||
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
|
||||
arm64. macOS validation is cross-compilation, not native runtime testing.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special migration is required for existing v2 configuration. To use the new
|
||||
features:
|
||||
|
||||
- Declare `notarius.llm_api_key` with the environment-variable name expected by
|
||||
the deployed Notarius profile, supplying its value through the existing
|
||||
environment or secrets-directory mechanism.
|
||||
- Enable checkpoint recording in Notarius configuration before setting
|
||||
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
|
||||
roots for service deployments. An optional `notarius.chunk_cache` overrides
|
||||
its ordinary chunk-plan policy.
|
||||
- Expect an initial checkpoint miss when moving from old run-local reference
|
||||
paths to stable session snapshots. No migration or deletion of old caches is
|
||||
required. Force refreshes the current input's plan, not the whole cache.
|
||||
- Use `narratio analyze SESSION` for analysis-only iteration, and
|
||||
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
|
||||
|
||||
See [Configuration](../config.md), the
|
||||
[Notarius integration contract](../integrations/notarius.md), and
|
||||
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
|
||||
for current details.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
|
||||
from machine configuration.
|
||||
- Added optional typed Notarius credentials, exact-name environment delivery,
|
||||
safe configuration projections, and captured-stream redaction for custom names.
|
||||
- Added Notarius resume and chunk-cache controls with force taking precedence.
|
||||
- Added verified, content-addressed session reference snapshots that survive
|
||||
ordinary run cleanup without becoming published artifacts.
|
||||
- Preserved relative-workspace support for snapshot delivery and confinement.
|
||||
- Kept credentials and cache-policy controls outside extract's semantic resume
|
||||
identity, so operational changes alone do not invalidate completed results.
|
||||
- Added an opt-in real Notarius integration test using a local provider to
|
||||
verify reuse, partial-failure recovery, and forced refresh without paid calls.
|
||||
33
docs/releases/v2.1.1.md
Normal file
33
docs/releases/v2.1.1.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Narratio v2.1.1
|
||||
|
||||
This release fixes configuration loading for publish rules that explicitly set
|
||||
requiredness while relying on a derived destination.
|
||||
|
||||
## Summary
|
||||
|
||||
Publish rules with a derivable source and omitted `dest` now load correctly
|
||||
whether `required` is omitted, explicitly true, or explicitly false. Previously,
|
||||
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
|
||||
diagnostic instead of accepting valid configuration.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a backward-compatible bookkeeping correction. Publish schemas,
|
||||
destination derivation, requiredness defaults, and YAML syntax support are
|
||||
unchanged. Explicit destinations and explicit false values remain intact.
|
||||
No ConfigKit upgrade or publication/storage behavior change is involved.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No special upgrade action or configuration migration is required. Operators
|
||||
need not add a derivable `dest`, remove explicit `required` values, or change
|
||||
flow-style YAML to block-style YAML. See the
|
||||
[configuration reference](../config.md#publish-configuration-summary).
|
||||
|
||||
## Changes
|
||||
|
||||
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
|
||||
while retaining field-level resolution records.
|
||||
- Added real load/validation and safe effective-configuration regression coverage
|
||||
for built-in and configured-artifact sources, explicit requiredness, and an
|
||||
explicit destination that must remain unchanged.
|
||||
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
@@ -0,0 +1,171 @@
|
||||
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
|
||||
|
||||
Status: Implemented; release target v2.1.1
|
||||
|
||||
## Summary
|
||||
|
||||
Narratio should successfully derive a publish destination when a valid
|
||||
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
|
||||
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
|
||||
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
|
||||
|
||||
This is a configuration-resolution bookkeeping defect, not a YAML formatting
|
||||
restriction. The fix should preserve the existing public configuration
|
||||
contract: operators may omit a derivable destination and may independently set
|
||||
`required` to either `true` or `false`.
|
||||
|
||||
## Observed Behavior
|
||||
|
||||
A pipeline containing a rule equivalent to the following fails during native
|
||||
configuration loading:
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
outputs: [{"source": "narratio.transcript.final", "required": true}]
|
||||
```
|
||||
|
||||
Narratio reports:
|
||||
|
||||
```text
|
||||
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
|
||||
```
|
||||
|
||||
The same failure is possible with ordinary block-style YAML when `required` is
|
||||
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
|
||||
cause of the defect.
|
||||
|
||||
The existing destination-derivation tests do not expose this case because
|
||||
their rules also omit `required`. Narratio then supplies the default value for
|
||||
`required` and records a change covering the complete `publish.outputs`
|
||||
sequence. That broader record happens to cover the destination mutation as
|
||||
well, masking the missing resolution record that occurs when `required` is
|
||||
already explicit.
|
||||
|
||||
## Expected Contract
|
||||
|
||||
The current configuration contract remains unchanged:
|
||||
|
||||
- `publish.outputs[].source` is required;
|
||||
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
|
||||
selected built-in or configured artifact source;
|
||||
- `publish.outputs[].required` defaults to `true`, but an operator may set it
|
||||
explicitly to either `true` or `false`; and
|
||||
- equivalent valid YAML representations have equivalent configuration
|
||||
semantics.
|
||||
|
||||
For example, both rules below should load and validate, retain their explicit
|
||||
requiredness, and receive the canonical derived destination:
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
outputs:
|
||||
- source: narratio.transcript.final
|
||||
required: true
|
||||
- source: narratio.transcript.final_markdown
|
||||
required: false
|
||||
```
|
||||
|
||||
## Root Cause
|
||||
|
||||
ConfigKit intentionally treats sequences as atomic when comparing values
|
||||
before and after application resolution. Changing a field within one
|
||||
`publish.outputs` entry is therefore observed as a change to the
|
||||
`/publish/outputs` sequence.
|
||||
|
||||
Narratio currently derives each missing destination by recording a
|
||||
normalization at the individual `publish.outputs[index].dest` path. That child
|
||||
record does not cover the atomic parent-sequence change. A separate
|
||||
container-level record is currently created only when Narratio also defaults
|
||||
an omitted `required` value, so configurations with explicit `required` values
|
||||
lose that incidental coverage.
|
||||
|
||||
## Recommended Fix
|
||||
|
||||
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
|
||||
atomic-sequence contract. Whenever resolution derives one or more destinations
|
||||
inside `publish.outputs`, the recorded resolution changes must cover the
|
||||
`/publish/outputs` container. Destination derivation must not depend on an
|
||||
unrelated `required` default being applied in the same list.
|
||||
|
||||
The implementation may retain more specific provenance where useful, but it
|
||||
must provide valid container-level coverage for the sequence mutation. Do not
|
||||
change ConfigKit's sequence comparison semantics merely to accommodate this
|
||||
call site.
|
||||
|
||||
## Scope
|
||||
|
||||
- Correct destination derivation for publish rules that explicitly set
|
||||
`required` and omit a derivable `dest`.
|
||||
- Preserve explicit `required: true` and `required: false` values.
|
||||
- Preserve canonical destination derivation for built-in transcript sources
|
||||
and configured artifact sources.
|
||||
- Add focused regression coverage through Narratio's real configuration load
|
||||
and validation boundary.
|
||||
- Prepare a backward-compatible patch release after the fix passes the normal
|
||||
repository validation.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
|
||||
destination derivation rules.
|
||||
- Requiring operators or configuration generators to supply `dest` when it is
|
||||
derivable.
|
||||
- Requiring operators to omit explicit `required: true` values.
|
||||
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
|
||||
- Changing ConfigKit's documented atomic treatment of sequences.
|
||||
- Adding storage, publication, or network integration behavior.
|
||||
|
||||
## Regression Coverage
|
||||
|
||||
Add one focused package-level behavioral test that loads and validates a
|
||||
pipeline with omitted destinations and explicit requiredness. A representative
|
||||
list should include both `required: true` and `required: false`, then assert
|
||||
that:
|
||||
|
||||
- loading and validation succeed;
|
||||
- each destination is derived from its source; and
|
||||
- each explicit `required` value is preserved.
|
||||
|
||||
Using the compact flow-style representation that exposed the integration
|
||||
failure is reasonable, while the existing block-style cases continue to cover
|
||||
omitted `required`. The test should exercise the public configuration behavior,
|
||||
not ConfigKit call choreography or Narratio's private recording mechanism.
|
||||
|
||||
No network service, object store, credential, or test double is required.
|
||||
|
||||
## Target End State
|
||||
|
||||
All valid publish rules with derivable sources behave consistently across
|
||||
these cases:
|
||||
|
||||
| `dest` | `required` | Result |
|
||||
| --- | --- | --- |
|
||||
| omitted | omitted | destination derived; required defaults to `true` |
|
||||
| omitted | `true` | destination derived; explicit `true` retained |
|
||||
| omitted | `false` | destination derived; explicit `false` retained |
|
||||
| explicit | either value | explicit destination and requiredness retained |
|
||||
|
||||
`narratio config validate`, `narratio config show`, and ordinary pipeline
|
||||
loading accept the corrected cases without an unrecorded-change diagnostic.
|
||||
No operator configuration migration or workaround is required.
|
||||
|
||||
## Release
|
||||
|
||||
This is a backward-compatible correction to documented configuration behavior
|
||||
and should be released as Narratio `v2.1.1`. The release note should identify
|
||||
the corrected explicit-required/derived-destination interaction and state that
|
||||
no special upgrade action is required.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- The focused regression test fails against v2.1.0 behavior and passes after
|
||||
the fix.
|
||||
- Publish rules with explicit `required: true` or `required: false` and omitted
|
||||
derivable destinations load and validate successfully.
|
||||
- Derived destinations and explicit requiredness appear correctly in the safe
|
||||
effective configuration.
|
||||
- Existing destination derivation, publish validation, imports, profiles, and
|
||||
configuration inspection tests remain green.
|
||||
- The complete repository validation and release-candidate checks pass.
|
||||
- The correction is published as the immutable `v2.1.1` tag with the required
|
||||
release note.
|
||||
@@ -56,7 +56,8 @@ Symptom:
|
||||
|
||||
Likely cause:
|
||||
|
||||
- stale field name, typo, invalid enum, or invalid duration/path format.
|
||||
- stale field name, typo, invalid enum, invalid duration/path format, or an
|
||||
unsupported local manifest schema/state marker.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -68,9 +69,39 @@ Safe fix:
|
||||
|
||||
- align config with [Configuration](./config.md) and the
|
||||
[maintained examples](../examples/README.md).
|
||||
- for a manifest schema error, move the unsupported local state aside and
|
||||
regenerate it; restore a verified current committed snapshot or republish
|
||||
healthy regenerated state when appropriate. Do not add version or stage
|
||||
markers by hand.
|
||||
- if an older manifest is rejected for its retired cleanup-obligation field,
|
||||
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, and use
|
||||
the cleanup stage. Do not treat manual JSON editing as the normal migration
|
||||
path.
|
||||
|
||||
Relevant reference: [Configuration](./config.md).
|
||||
|
||||
## Party schema version rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- configuration loading reports that `party.schema_version` is required or
|
||||
unsupported.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the campaign `party.yml` is an unversioned legacy document, or names a
|
||||
schema other than `narratio.party.v1`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- replace the file with one exact, single-document
|
||||
[`narratio.party.v1` party](./integrations/party.md); do not add a version
|
||||
marker to an otherwise incompatible legacy structure. The versioned party is
|
||||
the only configured roster authority.
|
||||
|
||||
Relevant reference: [Canonical Party Input](./integrations/party.md).
|
||||
|
||||
## Unexpected imported or profile value
|
||||
|
||||
Symptom:
|
||||
@@ -171,8 +202,9 @@ Safe fix:
|
||||
|
||||
- widen the bounded range to include the first reported stage, or recover that
|
||||
stage explicitly with `run-stage` before retrying. The failed check does not
|
||||
create a run record or modify the manifest. Narratio does not resume-validate
|
||||
excluded prefix stages, and stages after `--through` are not prerequisites.
|
||||
create a run-local directory or modify the manifest. Narratio does not
|
||||
resume-validate excluded prefix stages, and stages after `--through` are not
|
||||
prerequisites.
|
||||
|
||||
If prerequisite statuses are terminal but a selected stage reports a missing,
|
||||
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
||||
@@ -181,6 +213,40 @@ input validation.
|
||||
|
||||
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
||||
|
||||
## Notarius credential reference unavailable
|
||||
|
||||
Symptom:
|
||||
|
||||
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
|
||||
name for `pipeline.notarius.llm_api_key`, including during inspection or a
|
||||
run that would not execute extraction.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the named environment variable is absent or empty;
|
||||
- the selected secrets directory lacks a file with that exact name;
|
||||
- a process environment variable exists but is empty, so bootstrap correctly
|
||||
leaves it in place; or
|
||||
- the Narratio reference name does not match `api_key_env` in the selected
|
||||
Notarius/PromptKit profile.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- set the exact nonempty environment variable or provide an equivalently named
|
||||
file through the conventional or explicit secrets directory;
|
||||
- remove or set the field to `null` when that deployment needs no credential;
|
||||
and
|
||||
- use `config show`, `config sources`, and `config diff` to inspect the public
|
||||
reference name and ownership without exposing its value.
|
||||
|
||||
Narratio resolves selected credential references during ordinary pipeline
|
||||
loading, before stage selection or resume. Credential changes alone do not
|
||||
invalidate a completed extraction, but the current reference must still resolve
|
||||
for the command to proceed.
|
||||
|
||||
Relevant references: [Secrets](./config.md#secrets-handling) and
|
||||
[Notarius credential](./config.md#notarius-credential).
|
||||
|
||||
## Notarius executable missing
|
||||
|
||||
Symptom:
|
||||
@@ -257,6 +323,59 @@ Do not point Notarius directly at the original source path or edit the manifest
|
||||
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
||||
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius reference snapshot cache conflict
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports that a path beneath
|
||||
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
|
||||
non-regular or unsafe entry.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- stop commands for the affected session and inspect the exact reported path;
|
||||
- investigate unexpected links or special files as possible tampering; and
|
||||
- remove only the reported conflicting leaf or digest directory after
|
||||
confirming it belongs to the reconstructible reference cache.
|
||||
|
||||
A missing snapshot or corrupt regular snapshot is reconstructed automatically
|
||||
from currently verified prepared input. Do not recursively delete the session
|
||||
root, edit the manifest checksum, or remove cache entries while extraction is
|
||||
active.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius resume executes work instead of reusing it
|
||||
|
||||
Symptom:
|
||||
|
||||
- Narratio executes `extract`, and Notarius makes provider calls despite
|
||||
`pipeline.notarius.resume: true`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- checkpoint recording is disabled in the Notarius configuration;
|
||||
- Notarius is using a nonpersistent or different checkpoint root;
|
||||
- transcript/reference bytes, reference paths, pipeline/profile content,
|
||||
local-backend endpoint, selected lanes, or another checkpoint identity input
|
||||
changed; or
|
||||
- Narratio extraction was forced, which deliberately omits `--resume`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- validate the deployed Notarius configuration and configure explicit
|
||||
persistent chunk-plan and checkpoint roots;
|
||||
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
|
||||
and
|
||||
- retry without force only when reuse is intended.
|
||||
|
||||
Remember that Narratio skipping a completed `extract` stage is separate from
|
||||
Notarius reusing work inside an extraction that does execute. Narratio does not
|
||||
read the Notarius configuration to infer checkpoint enablement.
|
||||
|
||||
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius reference selector or generated-handoff collision
|
||||
|
||||
Symptom:
|
||||
@@ -282,29 +401,6 @@ Narratio validates selector structure and prepared evidence, while Notarius
|
||||
owns slot declarations, media compatibility, and generated-handoff conflicts.
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Atomic Notarius promotion unsupported
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||
before a durable bundle or temporary promotion tree is created.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio is running on an operating system other than Linux, macOS, or
|
||||
Windows, where the required atomic no-replace directory primitive has not
|
||||
been implemented and verified.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||
with a manual copy or move; the session manifest must never observe a partial
|
||||
or overwritten bundle.
|
||||
|
||||
This is an extraction-specific platform boundary, not a support statement for
|
||||
unrelated Narratio workflows. See
|
||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
@@ -352,18 +448,23 @@ Symptom:
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||
configured output contracts changed;
|
||||
- the pipeline ID or configured output contracts changed;
|
||||
- a configured prepared reference selector, source, path, checksum, or byte
|
||||
size changed;
|
||||
- the direct trimmed transcript identity or bytes changed; or
|
||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||
checksum no longer validates.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||
investigate it rather than replacing files manually.
|
||||
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
||||
and investigate it rather than replacing files manually.
|
||||
|
||||
Executable, config path, timeout, working directory, and Notarius credential
|
||||
changes are operational and do not by themselves invalidate reusable extraction
|
||||
evidence. A newly selected credential must nevertheless resolve during command
|
||||
configuration loading.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
@@ -395,8 +496,7 @@ Symptom:
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the per-artifact record is stale, missing, failed, unselected, malformed, or
|
||||
from the legacy aggregate-only manifest contract;
|
||||
- the per-artifact record is stale, missing, failed, unselected, or malformed;
|
||||
- a configured prompt/profile, dependency, input identity, output path, or
|
||||
effective variable changed; or
|
||||
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
|
||||
@@ -425,33 +525,82 @@ narratio analyze 2026-04-04 --artifacts session_recap
|
||||
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
|
||||
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
|
||||
|
||||
## Legacy aggregate analysis requires regeneration
|
||||
## Analysis manifest checkpoint failed
|
||||
|
||||
Symptom:
|
||||
|
||||
- a manifest from an older Narratio version reports aggregate analyze success
|
||||
and the old files are present, but configured artifact sources remain
|
||||
unavailable.
|
||||
- analyze reports that it could not save the session manifest after completing
|
||||
an artifact.
|
||||
|
||||
Likely cause:
|
||||
Likely causes:
|
||||
|
||||
- the manifest has no supported per-artifact analyze state. Aggregate output
|
||||
lists do not establish current configured-artifact authority.
|
||||
- the workspace is unwritable or out of space;
|
||||
- the manifest path or its parent directory has unsafe permissions or type; or
|
||||
- a transient local filesystem error interrupted atomic manifest replacement.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- regenerate the required artifacts. A partial selection makes only its
|
||||
targets and prerequisites eligible for current state; unselected legacy
|
||||
files intentionally remain unavailable. Run full analysis later when every
|
||||
enabled configured artifact must become current.
|
||||
- correct the reported filesystem or permission problem;
|
||||
- inspect `session artifacts` to identify the last durable current records; and
|
||||
- rerun analyze normally. Narratio reuses earlier successful checkpoints and
|
||||
retries the failed or non-current artifact and its required dependents.
|
||||
|
||||
Do not treat a newly created canonical artifact file as current when its
|
||||
checkpoint failed, and do not edit the manifest to advertise it. If the error
|
||||
recurs, preserve the manifest and run-local diagnostics before repairing the
|
||||
workspace.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#persistence-semantics).
|
||||
|
||||
## Built-in transcript or bounds source is unavailable
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze, extract, render, or publish says a built-in source is unavailable
|
||||
even though a file exists at its usual session path.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the producing stage has not succeeded, its output lacks the exact registered
|
||||
source, run ID, or checksum, its evidence is duplicated or stale, or the
|
||||
durable bytes no longer match the manifest checksum.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- do not copy or rename the file to make it discoverable. Rerun the producing
|
||||
stage to regenerate both the bytes and current manifest evidence, then retry
|
||||
the consumer.
|
||||
|
||||
## An older manifest schema is rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- Narratio rejects a local session manifest because `schema_version` is missing
|
||||
or unsupported; or
|
||||
- restore rejects a committed remote session manifest for the same reason.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the manifest predates the supported canonical state contract. Narratio
|
||||
rejects it at the schema boundary rather than interpreting aggregate output
|
||||
lists or incidental files as current authority.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- for unsupported local state, move the old manifest aside and regenerate, or
|
||||
restore a verified current committed snapshot when one is authoritative;
|
||||
- for unsupported committed remote state, regenerate healthy local state as
|
||||
needed and republish the session.
|
||||
|
||||
```bash
|
||||
narratio analyze 2026-04-04 --artifacts session_recap
|
||||
narratio analyze 2026-04-04
|
||||
narratio run 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
After current records exist, inspect them and publish explicitly. Do not delete
|
||||
the legacy files merely to influence selection; availability is manifest-owned.
|
||||
After current records exist, inspect them and publish explicitly. Do not copy
|
||||
old files into canonical paths merely to influence selection; availability is
|
||||
manifest-owned.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state).
|
||||
@@ -491,7 +640,9 @@ Symptom:
|
||||
Likely causes:
|
||||
|
||||
- missing `session.previous_session_id`;
|
||||
- previous artifact not restored/published for source session.
|
||||
- the selected previous-session commit does not declare the required canonical
|
||||
source as a `published_output`; or
|
||||
- the declared published object is unavailable or fails identity validation.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -549,6 +700,8 @@ Symptom:
|
||||
Likely cause:
|
||||
|
||||
- local durable files differ from remote restore sources.
|
||||
- a restore target or one of its ancestors is a symlink or another non-regular
|
||||
filesystem entry.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -559,7 +712,11 @@ narratio session restore 2026-04-04 --dry-run
|
||||
Safe fix:
|
||||
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite local.
|
||||
- rerun with `--force` only when remote state should overwrite an eligible
|
||||
regular local file; repair a symlink or other non-regular conflict first.
|
||||
|
||||
If the filesystem changed after the plan was printed, rerun restore after
|
||||
repairing the destination; installation performs the same safety check again.
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
@@ -652,7 +809,8 @@ Symptom:
|
||||
|
||||
Likely causes:
|
||||
|
||||
- invalid `pipeline.secrets.env_dir`;
|
||||
- missing, unreadable, or invalid explicit `--secrets-dir`;
|
||||
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
|
||||
- missing credential environment variables;
|
||||
- invalid S3 endpoint/bucket settings.
|
||||
|
||||
@@ -660,12 +818,12 @@ Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -la /path/to/secrets_dir
|
||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct secret-file path and permissions;
|
||||
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
|
||||
@@ -11,18 +11,26 @@ in the [configuration reference](../docs/config.md).
|
||||
WhisperX URL.
|
||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
|
||||
`conf.d` imports, a production default, and selectable production/testing
|
||||
overlays. It also demonstrates canonical-party artifact families and a
|
||||
testing-only disabled artifact. Validate it with:
|
||||
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
|
||||
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
|
||||
one family per `artifact-families/<key>.yml`, and the fixed
|
||||
`profiles/testing.yml` overlay. Production values live in the base. The
|
||||
bundle demonstrates an input-only artifact dependency, two character families
|
||||
sharing the campaign party, and a testing-only disabled artifact. Inspect it
|
||||
offline with:
|
||||
|
||||
```sh
|
||||
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
```
|
||||
|
||||
`config show` and `config sources` accept the same selectors and remain
|
||||
read-only.
|
||||
These inspection commands load configuration and resolve explicit credential
|
||||
references, but do not load a session or create runtime state. This bundle is
|
||||
intentionally local-storage and credential-reference free so the walkthrough
|
||||
needs no tools or cloud services.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
@@ -52,11 +60,20 @@ with the sample campaign and a compatible local- or S3-audio session.
|
||||
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
||||
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||
follows the Notarius v0.6 contract. Narratio derives the players projection
|
||||
from this party source; the campaign deliberately has no `players_file`.
|
||||
follows the Notarius v0.7 contract. Narratio derives the players projection
|
||||
from this party source.
|
||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
The examples contain environment-variable names but no credential values. They
|
||||
use fictional campaign content and reserved example domains.
|
||||
The examples contain environment-variable names but no credential values. Set
|
||||
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
|
||||
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
|
||||
examples that reference them. They use fictional campaign content and reserved
|
||||
example domains.
|
||||
|
||||
For a deployed bundle, keep credential files out of source control in a
|
||||
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
|
||||
environment-variable name, or select another directory with `--secrets-dir`.
|
||||
Static S3 credentials are optional: omit both S3 credential references to use
|
||||
the AWS SDK discovery chain.
|
||||
|
||||
@@ -7,9 +7,8 @@ workspace:
|
||||
# Optional: remove run-scoped workdir after successful publish commit.
|
||||
cleanup_after_publish: false
|
||||
|
||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||
# secrets:
|
||||
# env_dir: ./secrets
|
||||
# Credential files, when used, belong in a sibling secrets/ directory or in the
|
||||
# directory selected with --secrets-dir; they are not pipeline configuration.
|
||||
|
||||
storage:
|
||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||
@@ -24,8 +23,8 @@ storage:
|
||||
endpoint: ""
|
||||
force_path_style: false
|
||||
# Optional; defaults shown explicitly.
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||
@@ -94,7 +93,7 @@ audita:
|
||||
# Optional overrides; defaults shown explicitly where applicable.
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
base_url: ""
|
||||
model: ""
|
||||
@@ -129,13 +128,22 @@ trim:
|
||||
report: false
|
||||
|
||||
notarius:
|
||||
# Optional structured extraction between trim and render.
|
||||
# Optional structured extraction after render and before analyze.
|
||||
enabled: true
|
||||
binary: notarius
|
||||
# Optional environment-variable reference. Match this name to api_key_env in
|
||||
# the selected Notarius/PromptKit profile; never put the credential here.
|
||||
llm_api_key: NOTARIUS_LLM_API_KEY
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
working_directory: /usr/local/etc/notarius
|
||||
# Reuse compatible accepted Notarius checkpoints during executed extraction.
|
||||
# The Notarius config must enable recording and provide persistent cache roots.
|
||||
resume: true
|
||||
# auto uses or creates a source-addressed chunk plan. Forced Narratio
|
||||
# extraction overrides this with refresh and disables checkpoint loading.
|
||||
chunk_cache: auto
|
||||
# External campaign references use prepared Narratio source IDs. Omit an
|
||||
# optional binding when the selected Notarius pipeline does not need it.
|
||||
references:
|
||||
|
||||
@@ -8,8 +8,8 @@ storage:
|
||||
bucket: my-dnd-archive
|
||||
root_prefix: dnd
|
||||
region: us-east-1
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
@@ -57,7 +57,7 @@ seriatim:
|
||||
audita:
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
|
||||
@@ -1,23 +1,11 @@
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: ./scriptorium/config.yml
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_meta
|
||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
player_name: player.name
|
||||
class_summary: character.class_summary
|
||||
character_items:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_items
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||
member_dependencies: [character_meta]
|
||||
inputs:
|
||||
character_meta:
|
||||
source: narratio.member_artifact.character_meta
|
||||
@@ -0,0 +1,13 @@
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_meta
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
player_name: player.name
|
||||
class_summary: character.class_summary
|
||||
11
examples/production-testing/artifacts/player_handout.yml
Normal file
11
examples/production-testing/artifacts/player_handout.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
player_handout:
|
||||
enabled: true
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/player-handout.md
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
11
examples/production-testing/artifacts/session_recap.yml
Normal file
11
examples/production-testing/artifacts/session_recap.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/session-recap.md
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
@@ -10,6 +10,8 @@ audita:
|
||||
binary: audita
|
||||
modules: [glossary, grammar]
|
||||
output_schema: audita-v1
|
||||
model: narratio-production-model-placeholder
|
||||
validation_model: narratio-production-validator-placeholder
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
# Copyable production/testing pipeline entry point. Every fragment is named
|
||||
# explicitly; Narratio never scans conf.d automatically.
|
||||
composition:
|
||||
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
|
||||
# these explicit path and glob imports; Narratio never scans directories itself.
|
||||
configkit:
|
||||
imports:
|
||||
- conf.d/platform.yml
|
||||
- conf.d/storage.yml
|
||||
- conf.d/transcript.yml
|
||||
- conf.d/artifacts.yml
|
||||
- conf.d/publish.yml
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: profiles/testing.yml
|
||||
- path: conf.d/platform.yml
|
||||
- path: conf.d/storage.yml
|
||||
- path: conf.d/transcript.yml
|
||||
- path: conf.d/publish.yml
|
||||
- glob: artifacts/*.yml
|
||||
- glob: artifact-families/*.yml
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: ./scriptorium/config.yml
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
audita:
|
||||
model: narratio-production-model-placeholder
|
||||
validation_model: narratio-production-validator-placeholder
|
||||
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_meta:
|
||||
profile_id: production-placeholder
|
||||
character_items:
|
||||
profile_id: production-placeholder
|
||||
@@ -4,6 +4,10 @@ audita:
|
||||
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
profile_id: testing-placeholder
|
||||
player_handout:
|
||||
profile_id: testing-placeholder
|
||||
testing_notes:
|
||||
enabled: false
|
||||
output_path: artifacts/testing-notes.md
|
||||
|
||||
6
go.mod
6
go.mod
@@ -1,12 +1,15 @@
|
||||
module gitea.maximumdirect.net/eric/narratio
|
||||
|
||||
go 1.25.0
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/google/go-cmp v0.7.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -26,4 +29,5 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@@ -1,3 +1,5 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.0 h1:LgXEUjR0KsgwGBznOWa8FJTQ+iElctHtI1via5vKdu4=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.3.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
@@ -34,6 +36,12 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -84,17 +83,17 @@ func materializePlaceholders(req PolishRequest) error {
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -124,7 +123,7 @@ func writeJSONIfRequested(path string, payload any) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||
}
|
||||
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
19
internal/adapters/audita/generated_config.go
Normal file
19
internal/adapters/audita/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package audita
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -24,7 +23,8 @@ const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
||||
type SubprocessRunnerConfig struct {
|
||||
Binary string
|
||||
Timeout time.Duration
|
||||
LLMAPIKeyEnv string
|
||||
LLMAPIKey string
|
||||
LLMAPIKeyReference string
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
@@ -43,7 +43,8 @@ type SubprocessRunnerConfig struct {
|
||||
type SubprocessRunner struct {
|
||||
binary string
|
||||
timeout time.Duration
|
||||
llmAPIKeyEnv string
|
||||
llmAPIKey string
|
||||
llmAPIKeyReference string
|
||||
modules []string
|
||||
baseURL string
|
||||
model string
|
||||
@@ -62,7 +63,8 @@ type SubprocessRunner struct {
|
||||
func NewSubprocessRunnerFromConfigValues(
|
||||
binary string,
|
||||
timeout string,
|
||||
llmAPIKeyEnv string,
|
||||
llmAPIKey string,
|
||||
llmAPIKeyReference string,
|
||||
modules []string,
|
||||
baseURL string,
|
||||
model string,
|
||||
@@ -86,7 +88,8 @@ func NewSubprocessRunnerFromConfigValues(
|
||||
return NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: binary,
|
||||
Timeout: parsedTimeout,
|
||||
LLMAPIKeyEnv: llmAPIKeyEnv,
|
||||
LLMAPIKey: llmAPIKey,
|
||||
LLMAPIKeyReference: llmAPIKeyReference,
|
||||
Modules: modules,
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
@@ -110,6 +113,9 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
if cfg.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||
}
|
||||
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
|
||||
return nil, fmt.Errorf("audita credential value and reference must be provided together")
|
||||
}
|
||||
for i, module := range cfg.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||
@@ -152,7 +158,8 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
return &SubprocessRunner{
|
||||
binary: strings.TrimSpace(cfg.Binary),
|
||||
timeout: cfg.Timeout,
|
||||
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
|
||||
llmAPIKey: cfg.LLMAPIKey,
|
||||
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
|
||||
modules: modules,
|
||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
model: strings.TrimSpace(cfg.Model),
|
||||
@@ -195,16 +202,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
args := r.buildArgs(req, reqModules)
|
||||
|
||||
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
|
||||
credentialPresent := false
|
||||
credentialPresent := r.llmAPIKeyReference != ""
|
||||
env := map[string]string{}
|
||||
if credentialEnvVar != "" {
|
||||
credential, ok := os.LookupEnv(credentialEnvVar)
|
||||
if !ok || strings.TrimSpace(credential) == "" {
|
||||
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
|
||||
}
|
||||
env["AUDITA_LLM_API_KEY"] = credential
|
||||
credentialPresent = true
|
||||
if credentialPresent {
|
||||
env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
|
||||
}
|
||||
|
||||
if req.GeneratedConfigPath != "" {
|
||||
@@ -269,7 +270,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}, nil
|
||||
@@ -299,7 +300,7 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}
|
||||
@@ -375,10 +376,10 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func validateProcessedOutput(path string) error {
|
||||
@@ -425,7 +426,7 @@ func validateJSONFile(path string) error {
|
||||
}
|
||||
|
||||
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
data, err := fileops.ReadFileBounded(path, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||
}
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
@@ -31,7 +27,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: wrapper,
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary", "homophones", "glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
@@ -48,6 +45,7 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||
}
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
@@ -132,47 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
|
||||
t.Fatalf("error = %q, want missing credential guidance", err.Error())
|
||||
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
@@ -190,9 +172,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||
@@ -200,13 +179,13 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKeyEnv: "",
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKey: "",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -220,21 +199,18 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -249,19 +225,16 @@ func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -279,22 +252,20 @@ func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -310,25 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "super-secret") {
|
||||
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -341,22 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -369,22 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -397,22 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -425,22 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -453,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected binary validation error")
|
||||
}
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout parse error")
|
||||
}
|
||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKey: "synthetic-secret",
|
||||
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected base url validation error")
|
||||
|
||||
@@ -4,6 +4,8 @@ package notarius
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
||||
@@ -22,7 +24,9 @@ type ReferenceBinding struct {
|
||||
|
||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||
type RunRequest struct {
|
||||
Binary string
|
||||
Binary string
|
||||
// LLMAPIKey is an optional already-resolved credential reference.
|
||||
LLMAPIKey *configkit.Secret
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
InputPath string
|
||||
@@ -31,6 +35,8 @@ type RunRequest struct {
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
Timeout time.Duration
|
||||
Resume bool
|
||||
ChunkCache string
|
||||
References []ReferenceBinding
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,20 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
||||
if err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
var envOverrides map[string]string
|
||||
var sensitiveEnvNames []string
|
||||
if req.LLMAPIKey != nil {
|
||||
if !req.LLMAPIKey.IsResolved() {
|
||||
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||
}
|
||||
name := req.LLMAPIKey.EnvName()
|
||||
value := req.LLMAPIKey.Value()
|
||||
if value == "" {
|
||||
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||
}
|
||||
envOverrides = map[string]string{name: value}
|
||||
sensitiveEnvNames = []string{name}
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"run", req.PipelineID,
|
||||
@@ -59,18 +73,26 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
||||
"--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot,
|
||||
}
|
||||
if req.Resume {
|
||||
args = append(args, "--resume")
|
||||
}
|
||||
if req.ChunkCache != "" {
|
||||
args = append(args, "--chunk_cache", req.ChunkCache)
|
||||
}
|
||||
for _, reference := range references {
|
||||
args = append(args, "--reference", reference.Selector+"="+reference.Path)
|
||||
}
|
||||
args = append(args, "--json")
|
||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDirectory,
|
||||
Timeout: req.Timeout,
|
||||
DiagnosticOwner: "notarius",
|
||||
StdoutLogPath: req.ReceiptPath,
|
||||
StderrLogPath: req.LogPath,
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDirectory,
|
||||
Timeout: req.Timeout,
|
||||
DiagnosticOwner: "notarius",
|
||||
StdoutLogPath: req.ReceiptPath,
|
||||
StderrLogPath: req.LogPath,
|
||||
EnvOverrides: envOverrides,
|
||||
SensitiveEnvNames: sensitiveEnvNames,
|
||||
})
|
||||
baseResult := RunResult{
|
||||
ReceiptPath: req.ReceiptPath,
|
||||
@@ -143,6 +165,9 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
||||
if req.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("notarius timeout must be positive")
|
||||
}
|
||||
if !validChunkCacheMode(req.ChunkCache) {
|
||||
return nil, fmt.Errorf("notarius chunk cache mode must be empty, auto, bypass, or refresh")
|
||||
}
|
||||
for label, path := range map[string]string{
|
||||
"config": req.ConfigPath,
|
||||
"input": req.InputPath,
|
||||
@@ -201,6 +226,15 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
||||
return references, nil
|
||||
}
|
||||
|
||||
func validChunkCacheMode(mode string) bool {
|
||||
switch mode {
|
||||
case "", "auto", "bypass", "refresh":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type receiptDocument struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
@@ -248,7 +282,7 @@ func validateValidationSummaries(documents []validationSummaryDocument) ([]Valid
|
||||
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
||||
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
||||
}
|
||||
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) {
|
||||
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount < 0 || !validTerminalAction(document.TerminalAction) {
|
||||
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
||||
}
|
||||
summaries = append(summaries, ValidationSummary{
|
||||
@@ -671,7 +705,7 @@ func sumWarningOccurrences(values []WarningSummary) (int, error) {
|
||||
}
|
||||
|
||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
data, err := fileops.ReadFileBounded(path, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
|
||||
@@ -76,6 +77,127 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resume bool
|
||||
chunkCache string
|
||||
want []string
|
||||
}{
|
||||
{name: "resume", resume: true, want: []string{"--resume"}},
|
||||
{name: "auto", chunkCache: "auto", want: []string{"--chunk_cache", "auto"}},
|
||||
{name: "bypass", chunkCache: "bypass", want: []string{"--chunk_cache", "bypass"}},
|
||||
{name: "refresh", chunkCache: "refresh", want: []string{"--chunk_cache", "refresh"}},
|
||||
{name: "resume and mode", resume: true, chunkCache: "auto", want: []string{"--resume", "--chunk_cache", "auto"}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.Resume = test.resume
|
||||
req.ChunkCache = test.chunkCache
|
||||
var captured sharedsubprocess.RunRequest
|
||||
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||
}}
|
||||
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot,
|
||||
}
|
||||
want = append(want, test.want...)
|
||||
want = append(want, "--json")
|
||||
if !reflect.DeepEqual(captured.Args, want) {
|
||||
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
|
||||
for _, mode := range []string{"AUTO", "invalid", " auto "} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.ChunkCache = mode
|
||||
started := false
|
||||
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
started = true
|
||||
return sharedsubprocess.RunResult{}, nil
|
||||
}}
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "chunk cache mode") {
|
||||
t.Fatalf("Run() error = %v, want chunk cache validation failure", err)
|
||||
}
|
||||
if started {
|
||||
t.Fatal("subprocess started with invalid chunk cache mode")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerForwardsLoadedCredentialWithoutRereadingEnvironment(t *testing.T) {
|
||||
const envName = "NARRATIO_NOTARIUS_CUSTOM_LLM_KEY"
|
||||
const loadedValue = "loaded-notarius-credential"
|
||||
credential := loadTestCredential(t, envName, loadedValue)
|
||||
t.Setenv(envName, "changed-parent-credential")
|
||||
|
||||
req := validRunRequest(t)
|
||||
req.LLMAPIKey = credential
|
||||
var captured sharedsubprocess.RunRequest
|
||||
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||
}}
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(captured.EnvOverrides) != 1 || captured.EnvOverrides[envName] != loadedValue {
|
||||
t.Fatal("subprocess did not receive exactly the loaded credential override")
|
||||
}
|
||||
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{envName}) {
|
||||
t.Fatalf("sensitive environment names = %#v, want configured reference name", captured.SensitiveEnvNames)
|
||||
}
|
||||
wantArgs := []string{
|
||||
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot, "--json",
|
||||
}
|
||||
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRejectsInvalidProgrammaticCredentialBeforeLaunch(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
reference configkit.Secret
|
||||
}{
|
||||
{name: "unresolved", reference: configkit.SecretEnv("NARRATIO_NOTARIUS_UNRESOLVED_KEY")},
|
||||
{name: "empty", reference: configkit.SecretEnv("")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.LLMAPIKey = &test.reference
|
||||
started := false
|
||||
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
started = true
|
||||
return sharedsubprocess.RunResult{}, nil
|
||||
}}
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "resolved and nonempty") {
|
||||
t.Fatalf("Run() error = %v, want credential validation failure", err)
|
||||
}
|
||||
if started {
|
||||
t.Fatal("subprocess started with an invalid credential")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
referenceRoot := t.TempDir()
|
||||
@@ -172,16 +294,23 @@ func TestSubprocessRunnerRejectsInvalidReferencesBeforeLaunch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||
const envName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_KEY"
|
||||
const canary = "notarius-captured-stream-canary"
|
||||
const unrelated = "NARRATIO_NOTARIUS_UNRELATED_PARENT_KEY"
|
||||
req := validRunRequest(t)
|
||||
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
receiptFixture := req.ReceiptPath + ".fixture"
|
||||
data, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal(data, &receipt); err != nil {
|
||||
t.Fatalf("decode receipt fixture: %v", err)
|
||||
}
|
||||
receipt["credential_echo"] = canary
|
||||
writeJSONFile(t, receiptFixture, receipt)
|
||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||
t.Fatalf("Remove(receipt) error = %v", err)
|
||||
}
|
||||
@@ -193,27 +322,55 @@ func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T)
|
||||
script := writeShellScript(t, `#!/bin/sh
|
||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||
printf 'diagnostic stream\n' >&2
|
||||
if [ "$`+envName+`" != '`+canary+`' ]; then
|
||||
printf 'selected credential mismatch\n' >&2
|
||||
exit 8
|
||||
fi
|
||||
if [ -n "$`+unrelated+`" ]; then
|
||||
printf 'unrelated credential inherited\n' >&2
|
||||
exit 9
|
||||
fi
|
||||
printf '%s\n' "$`+envName+`" >&2
|
||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||
`)
|
||||
req.Binary = script
|
||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||
t.Setenv(unrelated, "unrelated-parent-credential")
|
||||
|
||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||
t.Fatal("receipt contains stderr output")
|
||||
assertRedactedCredentialBytes(t, receiptBytes, canary)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRedactsCredentialOnProcessFailure(t *testing.T) {
|
||||
const envName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_KEY"
|
||||
const canary = "notarius-failure-stream-canary"
|
||||
req := validRunRequest(t)
|
||||
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||
req.Binary = writeShellScript(t, `#!/bin/sh
|
||||
printf '%s\n' "$`+envName+`"
|
||||
printf '%s\n' "$`+envName+`" >&2
|
||||
exit 7
|
||||
`)
|
||||
_, err := NewSubprocessRunner().Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want process failure")
|
||||
}
|
||||
if strings.Contains(err.Error(), canary) || !strings.Contains(err.Error(), "<redacted>") {
|
||||
t.Fatal("process failure did not return redacted diagnostics")
|
||||
}
|
||||
assertRedactedCredentialFile(t, req.ReceiptPath, canary)
|
||||
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||
@@ -579,6 +736,56 @@ func validRunRequest(t *testing.T) RunRequest {
|
||||
}
|
||||
}
|
||||
|
||||
func loadTestCredential(t *testing.T, envName, value string) *configkit.Secret {
|
||||
t.Helper()
|
||||
t.Setenv(envName, value)
|
||||
type testConfig struct {
|
||||
Credential *configkit.Secret `yaml:"credential"`
|
||||
}
|
||||
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+envName+"\n")), configkit.Options[testConfig]{})
|
||||
if err != nil {
|
||||
t.Fatalf("load test credential: %v", err)
|
||||
}
|
||||
return result.Config.Credential
|
||||
}
|
||||
|
||||
func assertRedactedCredentialFile(t *testing.T, path, canary string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
assertRedactedCredentialBytes(t, data, canary)
|
||||
}
|
||||
|
||||
func assertRedactedCredentialBytes(t *testing.T, data []byte, canary string) {
|
||||
t.Helper()
|
||||
text := string(data)
|
||||
if strings.Contains(text, canary) || !strings.Contains(text, "<redacted>") {
|
||||
t.Fatal("captured stream did not contain the required redaction marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateValidationSummariesAcceptsReusedProducerWithoutAttempts(t *testing.T) {
|
||||
zero := 0
|
||||
summaries, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||
Stage: "chunk", Status: "complete", ProducerAttemptCount: &zero, TerminalAction: "accepted",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("validateValidationSummaries() error = %v", err)
|
||||
}
|
||||
if len(summaries) != 1 || summaries[0].ProducerAttemptCount != 0 {
|
||||
t.Fatalf("summaries = %#v, want one reused producer with zero attempts", summaries)
|
||||
}
|
||||
|
||||
negative := -1
|
||||
if _, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||
Stage: "chunk", Status: "complete", ProducerAttemptCount: &negative, TerminalAction: "accepted",
|
||||
}}); err == nil {
|
||||
t.Fatal("validateValidationSummaries() negative producer attempts error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -143,7 +142,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
|
||||
|
||||
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -155,17 +154,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -174,7 +173,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -186,17 +185,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
19
internal/adapters/scriptorium/generated_config.go
Normal file
19
internal/adapters/scriptorium/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -342,11 +342,11 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
|
||||
"render_format": payload.RenderFormat,
|
||||
"render_prompt_logged": payload.RenderPromptStore,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(path, data)
|
||||
}
|
||||
|
||||
func validateNonEmptyOutput(path string) error {
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
||||
}
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
||||
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -261,7 +260,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
|
||||
|
||||
func materializePlaceholders(req MergeRequest) error {
|
||||
if req.OutputMergedTranscriptPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
||||
}
|
||||
}
|
||||
@@ -272,22 +271,22 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
"input_transcript_paths": req.InputTranscriptPaths,
|
||||
"output_path": req.OutputMergedTranscriptPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -296,7 +295,7 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
|
||||
func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
if req.OutputTrimmedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -309,17 +308,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -328,7 +327,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
|
||||
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.OutputNormalizedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -344,22 +343,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.ReportPath != "" {
|
||||
payload["report_path"] = req.ReportPath
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -368,7 +367,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
if req.OutputRenderedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -385,17 +384,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
19
internal/adapters/seriatim/generated_config.go
Normal file
19
internal/adapters/seriatim/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package seriatim
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -553,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
|
||||
payload["coalesce_gap"] = *r.coalesceGap
|
||||
}
|
||||
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func buildTrimArgs(req TrimRequest) []string {
|
||||
@@ -605,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
||||
@@ -620,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
||||
"output_schema": outputSchema,
|
||||
"report_path": req.ReportPath,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||
@@ -638,7 +638,7 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
@@ -692,7 +692,7 @@ func validateNonEmptyTextFile(path string) error {
|
||||
}
|
||||
|
||||
func readSeriatimResult(path, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
||||
}
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
||||
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
||||
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
||||
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
@@ -19,14 +19,17 @@ import (
|
||||
type FakeBackend struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
Reads []FakeReadCall
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
Reads []FakeReadCall
|
||||
Deletes []string
|
||||
ExistsKeys []string
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
UploadErr error
|
||||
DeleteErr error
|
||||
ExistsErr error
|
||||
UploadHook func(FakeUploadCall) error
|
||||
DownloadHook func(FakeDownloadCall) error
|
||||
@@ -315,6 +318,25 @@ func fakeObjectETag(data []byte) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Delete removes one object. Removing an absent object succeeds.
|
||||
func (f *FakeBackend) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.DeleteErr != nil {
|
||||
return f.DeleteErr
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if normalizedKey == "" {
|
||||
return fmt.Errorf("delete object: key is required")
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.Deletes = append(f.Deletes, normalizedKey)
|
||||
delete(f.Objects, normalizedKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks object presence.
|
||||
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -323,8 +345,12 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if f.ExistsErr != nil {
|
||||
return false, f.ExistsErr
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
f.mu.Lock()
|
||||
f.ExistsKeys = append(f.ExistsKeys, normalizedKey)
|
||||
f.mu.Unlock()
|
||||
f.mu.RLock()
|
||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||
_, ok := f.Objects[normalizedKey]
|
||||
f.mu.RUnlock()
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
@@ -71,23 +71,55 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendDeleteIsIdempotent(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "locks/source.lock", Data: []byte("lock")})
|
||||
|
||||
for range 2 {
|
||||
if err := fake.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
}
|
||||
if _, ok := fake.Objects["locks/source.lock"]; ok {
|
||||
t.Fatal("Delete() left the object present")
|
||||
}
|
||||
if len(fake.Deletes) != 2 || fake.Deletes[0] != "locks/source.lock" || fake.Deletes[1] != "locks/source.lock" {
|
||||
t.Fatalf("Delete() calls = %#v, want two normalized keys", fake.Deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendDeleteHonorsContextAndInjectedError(t *testing.T) {
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
fake := &FakeBackend{}
|
||||
if err := fake.Delete(canceled, "locks/source.lock"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Delete() canceled error = %v, want context.Canceled", err)
|
||||
}
|
||||
|
||||
wantErr := errors.New("delete fail")
|
||||
fake.DeleteErr = wantErr
|
||||
if err := fake.Delete(context.Background(), "locks/source.lock"); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Delete() injected error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
|
||||
old := fake.Objects["locks.yml"].ETag
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||
fake.SeedObject(FakeObject{Key: "control.json", Data: []byte("old")})
|
||||
old := fake.Objects["control.json"].ETag
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
|
||||
t.Fatalf("locks object = %q, want successful replacement preserved", got)
|
||||
if got := string(fake.Objects["control.json"].Data); got != "new" {
|
||||
t.Fatalf("control object = %q, want successful replacement preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), DeleteErr: errors.New("delete fail"), ExistsErr: errors.New("exists fail")}
|
||||
|
||||
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||
t.Fatalf("List() error = %v, want list fail", err)
|
||||
@@ -100,6 +132,9 @@ func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||
}
|
||||
if err := fake.Delete(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "delete fail") {
|
||||
t.Fatalf("Delete() error = %v, want delete fail", err)
|
||||
}
|
||||
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ type ObjectStore interface {
|
||||
Download(ctx context.Context, key, localPath string) error
|
||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
Exists(ctx context.Context, key string) (bool, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ type s3API interface {
|
||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
|
||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||
}
|
||||
|
||||
@@ -36,8 +37,12 @@ type s3ClientOptions struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
ForcePathStyle bool
|
||||
AccessKeyID string
|
||||
SecretKey string
|
||||
Credentials *s3StaticCredentials
|
||||
}
|
||||
|
||||
type s3StaticCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
@@ -45,11 +50,11 @@ var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error)
|
||||
if strings.TrimSpace(opts.Region) != "" {
|
||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||
}
|
||||
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
||||
if opts.Credentials != nil {
|
||||
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
strings.TrimSpace(opts.AccessKeyID),
|
||||
strings.TrimSpace(opts.SecretKey),
|
||||
opts.Credentials.AccessKeyID,
|
||||
opts.Credentials.SecretAccessKey,
|
||||
"",
|
||||
),
|
||||
))
|
||||
@@ -74,14 +79,24 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||
}
|
||||
if (cfg.AccessKeyID == nil) != (cfg.SecretAccessKey == nil) {
|
||||
return nil, fmt.Errorf("storage.s3 access key ID and secret access key references must be configured together")
|
||||
}
|
||||
if cfg.AccessKeyID != nil && (!cfg.AccessKeyID.IsResolved() || !cfg.SecretAccessKey.IsResolved()) {
|
||||
return nil, fmt.Errorf("storage.s3 configured credential references must be resolved")
|
||||
}
|
||||
|
||||
client, err := newS3Client(ctx, s3ClientOptions{
|
||||
clientOptions := s3ClientOptions{
|
||||
Region: cfg.Region,
|
||||
Endpoint: cfg.Endpoint,
|
||||
ForcePathStyle: cfg.ForcePathStyle,
|
||||
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
||||
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
||||
})
|
||||
}
|
||||
if cfg.AccessKeyID != nil && cfg.SecretAccessKey != nil {
|
||||
clientOptions.Credentials = &s3StaticCredentials{
|
||||
AccessKeyID: cfg.AccessKeyID.Value(), SecretAccessKey: cfg.SecretAccessKey.Value(),
|
||||
}
|
||||
}
|
||||
client, err := newS3Client(ctx, clientOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||
}
|
||||
@@ -92,26 +107,6 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
||||
}, nil
|
||||
}
|
||||
|
||||
func s3CredentialFromEnv(envVarName string) string {
|
||||
name := strings.TrimSpace(envVarName)
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
value, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func orDefaultEnvName(name, fallback string) string {
|
||||
trimmed := strings.TrimSpace(name)
|
||||
if trimmed == "" {
|
||||
return fallback
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
// List returns objects under prefix.
|
||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
@@ -300,6 +295,25 @@ func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key stri
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Delete removes one object. S3 deletion is idempotent when the key is absent.
|
||||
func (b *S3Backend) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if normalizedKey == "" {
|
||||
return fmt.Errorf("delete object: key is required")
|
||||
}
|
||||
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete object %q: %w", normalizedKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks whether one object key exists.
|
||||
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
@@ -29,16 +30,18 @@ type fakeS3API struct {
|
||||
getETag *string
|
||||
getLastModified *time.Time
|
||||
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
deleteErr error
|
||||
|
||||
headErr error
|
||||
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastLists []*s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastLists []*s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastDelete *s3.DeleteObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
}
|
||||
|
||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||
@@ -116,6 +119,14 @@ func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ..
|
||||
return f.putOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) DeleteObject(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
|
||||
f.lastDelete = params
|
||||
if f.deleteErr != nil {
|
||||
return nil, f.deleteErr
|
||||
}
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||
f.lastHead = params
|
||||
if f.headErr != nil {
|
||||
@@ -177,14 +188,14 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
||||
client := &fakeS3API{
|
||||
getBody: io.NopCloser(strings.NewReader("locks")),
|
||||
getBody: io.NopCloser(strings.NewReader("state")),
|
||||
getSize: int64Ptr(5),
|
||||
getETag: strPtr(`"generation"`),
|
||||
getLastModified: &lastModified,
|
||||
}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
|
||||
info, body, err := backend.Read(context.Background(), `sessions\control.json`)
|
||||
if err != nil {
|
||||
t.Fatalf("Read() error = %v", err)
|
||||
}
|
||||
@@ -193,10 +204,10 @@ func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||
if readErr != nil || closeErr != nil {
|
||||
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
||||
}
|
||||
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||
if info.Key != "sessions/control.json" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
||||
}
|
||||
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
|
||||
if string(data) != "state" || client.lastGet == nil || *client.lastGet.Key != "sessions/control.json" {
|
||||
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
||||
}
|
||||
}
|
||||
@@ -238,14 +249,14 @@ func TestS3BackendUploadAndExists(t *testing.T) {
|
||||
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
||||
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
||||
}
|
||||
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||
if !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
@@ -275,11 +286,44 @@ func TestS3BackendExistsNotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendDeleteMapsRequestAndErrors(t *testing.T) {
|
||||
client := &fakeS3API{}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
if err := backend.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
if client.lastDelete == nil || *client.lastDelete.Bucket != "bucket-1" || *client.lastDelete.Key != "locks/source.lock" {
|
||||
t.Fatalf("DeleteObject input = %#v, want normalized bucket and key", client.lastDelete)
|
||||
}
|
||||
|
||||
wantErr := errors.New("permission denied")
|
||||
client.deleteErr = wantErr
|
||||
err := backend.Delete(context.Background(), "locks/source.lock")
|
||||
if !errors.Is(err, wantErr) || !strings.Contains(err.Error(), "locks/source.lock") {
|
||||
t.Fatalf("Delete() error = %v, want contextual backend error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := &fakeS3API{}
|
||||
err := (&S3Backend{bucket: "bucket-1", client: client}).Delete(ctx, "locks/source.lock")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Delete() error = %v, want context.Canceled", err)
|
||||
}
|
||||
if client.lastDelete != nil {
|
||||
t.Fatal("Delete() called S3 after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
||||
accessKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_ACCESS", "id-123")
|
||||
secretKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_SECRET", "secret-abc")
|
||||
t.Setenv("NARRATIO_S3_TEST_ACCESS", "changed-id")
|
||||
t.Setenv("NARRATIO_S3_TEST_SECRET", "changed-secret")
|
||||
|
||||
var got s3ClientOptions
|
||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
@@ -288,10 +332,12 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
}
|
||||
|
||||
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
Endpoint: "http://localhost:9000",
|
||||
ForcePathStyle: true,
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
Endpoint: "http://localhost:9000",
|
||||
ForcePathStyle: true,
|
||||
AccessKeyID: accessKey,
|
||||
SecretAccessKey: secretKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
@@ -302,7 +348,7 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||
}
|
||||
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
||||
if got.Credentials == nil || got.Credentials.AccessKeyID != "id-123" || got.Credentials.SecretAccessKey != "secret-abc" {
|
||||
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||
}
|
||||
}
|
||||
@@ -314,7 +360,21 @@ func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
func TestNewS3BackendFromConfigRejectsIncompleteOrUnresolvedCredentials(t *testing.T) {
|
||||
unresolved := configkit.SecretEnv("NARRATIO_UNRESOLVED_S3_KEY")
|
||||
for name, cfg := range map[string]config.StorageS3Config{
|
||||
"partial": {Bucket: "my-archive", AccessKeyID: &unresolved},
|
||||
"unresolved": {Bucket: "my-archive", AccessKeyID: &unresolved, SecretAccessKey: &unresolved},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := NewS3BackendFromConfig(context.Background(), cfg); err == nil {
|
||||
t.Fatal("NewS3BackendFromConfig() succeeded, want credential error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigUsesDefaultProviderWithoutCredentialReferences(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
|
||||
@@ -325,19 +385,30 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
||||
SecretKeyEnv: "MISSING_SECRET_KEY",
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "" || got.SecretKey != "" {
|
||||
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
||||
if got.Credentials != nil {
|
||||
t.Fatalf("client options credentials = %#v, want default provider chain", got.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedTestSecret(t *testing.T, name, value string) *configkit.Secret {
|
||||
t.Helper()
|
||||
t.Setenv(name, value)
|
||||
type document struct {
|
||||
Secret *configkit.Secret `yaml:"secret"`
|
||||
}
|
||||
result, err := configkit.Load(configkit.Bytes("test.yml", []byte("secret: "+name+"\n")), configkit.Options[document]{})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve test secret: %v", err)
|
||||
}
|
||||
return result.Config.Secret
|
||||
}
|
||||
|
||||
func strPtr(v string) *string { return &v }
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
func boolPtr(v bool) *bool { return &v }
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
|
||||
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||
store := &FakeBackend{}
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/runs/run-123/metadata.txt", Data: []byte("run-123\n")})
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
||||
t.Fatalf("Glob(before) error = %v", err)
|
||||
}
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", pattern)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||
}
|
||||
@@ -58,7 +58,7 @@ func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
||||
sentinel := errors.New("object missing")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||
if err == nil {
|
||||
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ type logWriters struct {
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
|
||||
limits chan *captureLimitError
|
||||
limits chan error
|
||||
mu sync.Mutex
|
||||
limit *captureLimitError
|
||||
stdout *diagnosticWriter
|
||||
@@ -92,7 +92,7 @@ type diagnosticWriter struct {
|
||||
}
|
||||
|
||||
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
logs := &logWriters{limits: make(chan error, 1)}
|
||||
cleanStdout := cleanLogPath(stdoutPath)
|
||||
cleanStderr := cleanLogPath(stderrPath)
|
||||
|
||||
@@ -221,7 +221,7 @@ func (w *diagnosticWriter) reachLimit() error {
|
||||
return limit
|
||||
}
|
||||
|
||||
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
|
||||
func (l *logWriters) Limits() <-chan error { return l.limits }
|
||||
|
||||
func (l *logWriters) Limit() *captureLimitError {
|
||||
l.mu.Lock()
|
||||
@@ -255,7 +255,7 @@ func openDiagnosticFile(path string) (*os.File, error) {
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||
}
|
||||
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||
}
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
// Package subprocess provides reusable process execution and generated-config helpers.
|
||||
// Package subprocess provides internal process execution and Narratio-specific
|
||||
// subprocess policy. Its private execution mechanism owns process lifecycle;
|
||||
// the Run boundary owns environments, diagnostics, redaction, and errors.
|
||||
package subprocess
|
||||
|
||||
91
internal/adapters/subprocess/execution.go
Normal file
91
internal/adapters/subprocess/execution.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
type executionReason uint8
|
||||
|
||||
const (
|
||||
executionNotStarted executionReason = iota
|
||||
executionNaturalExit
|
||||
executionCanceled
|
||||
executionDeadline
|
||||
executionAborted
|
||||
)
|
||||
|
||||
type executionOutcome struct {
|
||||
reason executionReason
|
||||
startedAt time.Time
|
||||
completedAt time.Time
|
||||
exitCode int
|
||||
startErr error
|
||||
waitErr error
|
||||
contextErr error
|
||||
abortErr error
|
||||
cleanupErr error
|
||||
}
|
||||
|
||||
func executeOwnedCommand(ctx context.Context, cmd *exec.Cmd, abort <-chan error) (outcome executionOutcome) {
|
||||
outcome = executionOutcome{startedAt: time.Now().UTC(), exitCode: -1}
|
||||
defer func() { outcome.completedAt = time.Now().UTC() }()
|
||||
if outcome.contextErr = ctx.Err(); outcome.contextErr != nil {
|
||||
outcome.reason = executionNotStarted
|
||||
return outcome
|
||||
}
|
||||
tree, err := newOwnedProcessTree()
|
||||
if err != nil {
|
||||
outcome.reason, outcome.startErr = executionNotStarted, err
|
||||
return outcome
|
||||
}
|
||||
defer func() { outcome.cleanupErr = joinErrors(outcome.cleanupErr, tree.Dispose()) }()
|
||||
cmd.WaitDelay = forcefulTerminationWait
|
||||
if outcome.startErr = tree.Start(cmd); outcome.startErr != nil {
|
||||
outcome.reason = executionNotStarted
|
||||
return outcome
|
||||
}
|
||||
waitCh := make(chan error, 1)
|
||||
go func() { waitCh <- cmd.Wait() }()
|
||||
select {
|
||||
case outcome.waitErr = <-waitCh:
|
||||
outcome.reason = executionNaturalExit
|
||||
case <-ctx.Done():
|
||||
outcome.contextErr = ctx.Err()
|
||||
if outcome.contextErr == context.DeadlineExceeded {
|
||||
outcome.reason = executionDeadline
|
||||
} else {
|
||||
outcome.reason = executionCanceled
|
||||
}
|
||||
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||
case outcome.abortErr = <-abort:
|
||||
outcome.reason = executionAborted
|
||||
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||
}
|
||||
if cmd.ProcessState != nil {
|
||||
outcome.exitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
func terminateAndReap(tree ownedProcessTree, waitCh <-chan error) (waitErr, cleanupErr error) {
|
||||
cleanupErr = tree.TerminateGracefully()
|
||||
timer := time.NewTimer(gracefulTerminationWait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
case <-timer.C:
|
||||
}
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||
defer forcefulTimer.Stop()
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, cleanupErr
|
||||
case <-forcefulTimer.C:
|
||||
return nil, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
}
|
||||
54
internal/adapters/subprocess/execution_test.go
Normal file
54
internal/adapters/subprocess/execution_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTerminateAndReapPreservesWaitAndCleanupErrors(t *testing.T) {
|
||||
waitErr := errors.New("wait failed")
|
||||
gracefulErr := errors.New("graceful termination failed")
|
||||
forcefulErr := errors.New("forceful termination failed")
|
||||
tree := &recordingProcessTree{
|
||||
gracefulErr: gracefulErr,
|
||||
forcefulErr: forcefulErr,
|
||||
}
|
||||
waitCh := make(chan error, 1)
|
||||
waitCh <- waitErr
|
||||
|
||||
gotWaitErr, gotCleanupErr := terminateAndReap(tree, waitCh)
|
||||
if !errors.Is(gotWaitErr, waitErr) {
|
||||
t.Fatalf("wait error = %v, want %v", gotWaitErr, waitErr)
|
||||
}
|
||||
for _, want := range []error{gracefulErr, forcefulErr} {
|
||||
if !errors.Is(gotCleanupErr, want) {
|
||||
t.Fatalf("cleanup error = %v, want wrapped %v", gotCleanupErr, want)
|
||||
}
|
||||
}
|
||||
if tree.gracefulCalls != 1 || tree.forcefulCalls != 1 {
|
||||
t.Fatalf("termination calls = graceful %d, forceful %d; want 1 each", tree.gracefulCalls, tree.forcefulCalls)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingProcessTree struct {
|
||||
gracefulErr error
|
||||
forcefulErr error
|
||||
|
||||
gracefulCalls int
|
||||
forcefulCalls int
|
||||
}
|
||||
|
||||
func (*recordingProcessTree) Start(*exec.Cmd) error { return nil }
|
||||
|
||||
func (tree *recordingProcessTree) TerminateGracefully() error {
|
||||
tree.gracefulCalls++
|
||||
return tree.gracefulErr
|
||||
}
|
||||
|
||||
func (tree *recordingProcessTree) TerminateForcefully() error {
|
||||
tree.forcefulCalls++
|
||||
return tree.forcefulErr
|
||||
}
|
||||
|
||||
func (*recordingProcessTree) Dispose() error { return nil }
|
||||
@@ -1,9 +1,7 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
@@ -22,39 +20,6 @@ type ownedProcessTree interface {
|
||||
Dispose() error
|
||||
}
|
||||
|
||||
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, nil, nil, nil
|
||||
case <-ctx.Done():
|
||||
ctxErr = ctx.Err()
|
||||
case captureLimit = <-captureLimits:
|
||||
}
|
||||
|
||||
cleanupErr = tree.TerminateGracefully()
|
||||
gracefulTimer := time.NewTimer(gracefulTerminationWait)
|
||||
defer gracefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
// The leader may exit before descendants finish graceful shutdown.
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-gracefulTimer.C:
|
||||
}
|
||||
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||
defer forcefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-forcefulTimer.C:
|
||||
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
}
|
||||
|
||||
func joinErrors(errs ...error) error {
|
||||
filtered := make([]error, 0, len(errs))
|
||||
for _, err := range errs {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build linux || darwin || windows
|
||||
//go:build linux || darwin
|
||||
|
||||
package subprocess
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
type windowsProcessTree struct {
|
||||
job windows.Handle
|
||||
}
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
return &windowsProcessTree{}, nil
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
||||
job, err := windows.CreateJobObject(nil, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create job object: %w", err)
|
||||
}
|
||||
|
||||
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
||||
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return fmt.Errorf("configure job object: %w", err)
|
||||
}
|
||||
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return err
|
||||
}
|
||||
|
||||
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
|
||||
if err == nil {
|
||||
err = windows.AssignProcessToJobObject(job, process)
|
||||
_ = windows.CloseHandle(process)
|
||||
}
|
||||
if err == nil {
|
||||
err = resumeInitialThread(uint32(cmd.Process.Pid))
|
||||
}
|
||||
if err != nil {
|
||||
killErr := cmd.Process.Kill()
|
||||
waitErr := cmd.Wait()
|
||||
_ = windows.CloseHandle(job)
|
||||
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
|
||||
}
|
||||
|
||||
tree.job = job
|
||||
return nil
|
||||
}
|
||||
|
||||
func resumeInitialThread(processID uint32) error {
|
||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot initial thread: %w", err)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(snapshot) }()
|
||||
|
||||
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
||||
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
for {
|
||||
if entry.OwnerProcessID != processID {
|
||||
// Keep enumerating until the suspended process's only initial thread
|
||||
// is found.
|
||||
} else {
|
||||
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
||||
if openErr != nil {
|
||||
return fmt.Errorf("open initial thread: %w", openErr)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(thread) }()
|
||||
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
||||
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
||||
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
||||
break
|
||||
}
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateGracefully() error {
|
||||
// Windows jobs have no portable graceful signal. Terminating the owned job
|
||||
// is the safe fallback and prevents a descendant from escaping cleanup.
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateForcefully() error {
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Dispose() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
err := windows.CloseHandle(tree.job)
|
||||
tree.job = 0
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) terminate() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
return windows.TerminateJobObject(tree.job, 1)
|
||||
}
|
||||
@@ -7,9 +7,6 @@ import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// RunRequest defines a subprocess invocation.
|
||||
@@ -60,63 +57,23 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
}
|
||||
defer logs.Close()
|
||||
|
||||
tree, err := newOwnedProcessTree()
|
||||
if err != nil {
|
||||
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(req.Executable, req.Args...)
|
||||
cmd.Dir = req.WorkingDir
|
||||
cmd.Env = childEnv
|
||||
cmd.Stdout = logs.Stdout
|
||||
cmd.Stderr = logs.Stderr
|
||||
// Streaming capture uses pipes. Bound their lifetime when a leader exits
|
||||
// while a descendant still holds a stream descriptor.
|
||||
cmd.WaitDelay = forcefulTerminationWait
|
||||
|
||||
started := time.Now().UTC()
|
||||
result := RunResult{
|
||||
ExitCode: -1,
|
||||
StartedAt: started,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
}
|
||||
|
||||
if err := runCtx.Err(); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("command was not started: %w", err)
|
||||
}
|
||||
|
||||
if err := tree.Start(cmd); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
||||
}
|
||||
|
||||
waitCh := make(chan error, 1)
|
||||
go func() { waitCh <- cmd.Wait() }()
|
||||
|
||||
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
|
||||
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
|
||||
cleanupErr = joinErrors(cleanupErr, logs.Flush())
|
||||
outcome := executeOwnedCommand(runCtx, cmd, logs.Limits())
|
||||
cleanupErr := joinErrors(outcome.cleanupErr, logs.Flush())
|
||||
captureLimit, _ := outcome.abortErr.(*captureLimitError)
|
||||
if captureLimit == nil {
|
||||
captureLimit = logs.Limit()
|
||||
}
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result := RunResult{ExitCode: outcome.exitCode, StartedAt: outcome.startedAt, CompletedAt: outcome.completedAt, StdoutLogPath: req.StdoutLogPath, StderrLogPath: req.StderrLogPath}
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
if cmd.ProcessState != nil {
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
result.TimedOut = outcome.reason == executionDeadline
|
||||
result.Canceled = outcome.reason == executionCanceled
|
||||
|
||||
if ctxErr == context.DeadlineExceeded {
|
||||
result.TimedOut = true
|
||||
}
|
||||
if ctxErr == context.Canceled && !result.TimedOut {
|
||||
result.Canceled = true
|
||||
}
|
||||
|
||||
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
|
||||
if outcome.reason == executionNaturalExit && outcome.waitErr == nil && cleanupErr == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -124,48 +81,28 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
diagnostics := buildDiagnostics(req, result, stderrTail)
|
||||
|
||||
if captureLimit != nil {
|
||||
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
|
||||
if cause := joinErrors(outcome.waitErr, cleanupErr); cause != nil {
|
||||
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
||||
}
|
||||
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
||||
}
|
||||
if result.TimedOut {
|
||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if result.Canceled {
|
||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
if exitErr, ok := outcome.waitErr.(*exec.ExitError); ok {
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
||||
func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
|
||||
data, err := yaml.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal yaml for %q: %w", path, err)
|
||||
if outcome.startErr != nil {
|
||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, outcome.startErr)
|
||||
}
|
||||
if err := WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write yaml %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
|
||||
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("write file: path is required")
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write file %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(outcome.waitErr, outcome.contextErr, outcome.abortErr, cleanupErr))
|
||||
}
|
||||
|
||||
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
||||
|
||||
@@ -8,14 +8,11 @@ import (
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestRunSuccessCapturesStdoutStderr(t *testing.T) {
|
||||
@@ -139,9 +136,6 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
@@ -165,15 +159,15 @@ func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||
StdoutLogPath: stdoutPath,
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
||||
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
data, readErr := os.ReadFile(targetPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadFile(target) error = %v", readErr)
|
||||
}
|
||||
if string(data) != original {
|
||||
t.Fatalf("target content = %q, want %q", data, original)
|
||||
if string(data) == original {
|
||||
t.Fatal("diagnostic target was not replaced")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -382,7 +376,7 @@ func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
exactLogs := &logWriters{limits: make(chan error, 1)}
|
||||
var exactOutput bytes.Buffer
|
||||
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
||||
if _, err := exact.Write([]byte("abcde")); err != nil {
|
||||
@@ -398,7 +392,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
t.Fatal("exact write recorded a capture limit")
|
||||
}
|
||||
|
||||
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
cappedLogs := &logWriters{limits: make(chan error, 1)}
|
||||
var cappedOutput bytes.Buffer
|
||||
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
||||
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
||||
@@ -416,7 +410,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
logs := &logWriters{limits: make(chan error, 1)}
|
||||
var output bytes.Buffer
|
||||
secret := "credential-value"
|
||||
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
||||
@@ -592,59 +586,6 @@ func TestRunSharedStdoutStderrLogPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteYAMLAtomic(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.generated.yml")
|
||||
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"name": "narratio", "stage": "merge"}, 0o644); err != nil {
|
||||
t.Fatalf("WriteYAMLAtomic() error = %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read yaml: %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := yaml.Unmarshal(data, &got); err != nil {
|
||||
t.Fatalf("yaml unmarshal: %v", err)
|
||||
}
|
||||
if got["name"] != "narratio" {
|
||||
t.Fatalf("name = %#v, want narratio", got["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteYAMLAtomicOverwriteNoTempResidue(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.generated.yml")
|
||||
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "one"}, 0o644); err != nil {
|
||||
t.Fatalf("first write: %v", err)
|
||||
}
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "two"}, 0o644); err != nil {
|
||||
t.Fatalf("second write: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read yaml: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "two") {
|
||||
t.Fatalf("yaml = %q, want overwritten value", string(data))
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if strings.Contains(name, ".tmp-") {
|
||||
t.Fatalf("temp file residue found: %q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_SUBPROCESS_HELPER") != "1" {
|
||||
return
|
||||
|
||||
@@ -38,11 +38,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
|
||||
var capturedStages []string
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -52,7 +48,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{
|
||||
"run-stage", "publish", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
@@ -62,6 +58,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -154,7 +151,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=4 skipped=8") {
|
||||
if !strings.Contains(out.String(), "executed=5 skipped=9") {
|
||||
t.Fatalf("output = %q, want extract reconsidered and legacy analyze plus delivery rebuilt", out.String())
|
||||
}
|
||||
}
|
||||
@@ -165,11 +162,7 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -182,10 +175,11 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -206,18 +200,14 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{
|
||||
"analyze",
|
||||
"2026-05-03",
|
||||
@@ -228,6 +218,7 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -281,13 +272,18 @@ func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"analyze", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if got := stderr.String(); !strings.Contains(got, "analyze: no pipeline config path provided") ||
|
||||
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
@@ -299,11 +295,7 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -317,10 +309,11 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -383,13 +376,18 @@ func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"publish", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if got := stderr.String(); !strings.Contains(got, "publish: no pipeline config path provided") ||
|
||||
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestExecuteStagesRetriesFromLastAnalyzeCheckpoint(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: checkpointRecoveryArtifacts("original")}
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim"} {
|
||||
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
analyze := mustSelectStage(t, "analyze")
|
||||
|
||||
baselineRunner := &analyzeCheckpointRunner{}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: baselineRunner},
|
||||
}); err != nil {
|
||||
t.Fatalf("baseline executeStages() error = %v", err)
|
||||
}
|
||||
baseline, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("load baseline manifest: %v", err)
|
||||
}
|
||||
baseline.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||
baseline.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), baseline); err != nil {
|
||||
t.Fatalf("save delivery baseline: %v", err)
|
||||
}
|
||||
upstream := cloneStageRecords(baseline, "prepare", "transcribe", "merge", "polish", "normalize", "trim")
|
||||
unrelated := baseline.Stages["analyze"].AnalyzeArtifacts["unrelated"]
|
||||
|
||||
cfg.Pipeline.Scriptorium.Artifacts = checkpointRecoveryArtifacts("revised")
|
||||
failure := errors.New("injected quest-log failure")
|
||||
failingRunner := &analyzeCheckpointRunner{FailPrompt: "quest-log-revised", Err: failure}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: failingRunner},
|
||||
}); !errors.Is(err, failure) {
|
||||
t.Fatalf("executeStages() error = %v, want quest-log failure", err)
|
||||
}
|
||||
if want := []string{"chronicle-revised", "encounters-revised", "quest-log-revised"}; !slices.Equal(failingRunner.Calls, want) {
|
||||
t.Fatalf("failed-run calls = %#v, want %#v", failingRunner.Calls, want)
|
||||
}
|
||||
|
||||
failed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
failedAnalyze := failed.Stages["analyze"]
|
||||
if failedAnalyze.Status != manifest.StatusFailed || len(failedAnalyze.Outputs) != 0 {
|
||||
t.Fatalf("aggregate analyze = %#v, want failed without outputs", failedAnalyze)
|
||||
}
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "encounters", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "quest_log", manifest.AnalyzeArtifactFailed)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "player_handout", manifest.AnalyzeArtifactStale)
|
||||
gotUnrelated := failedAnalyze.AnalyzeArtifacts["unrelated"]
|
||||
if diff := cmp.Diff(unrelated, gotUnrelated); diff != "" {
|
||||
t.Fatalf("unrelated artifact changed (-want +got):\n%s", diff)
|
||||
}
|
||||
for _, name := range []string{"publish", "notify"} {
|
||||
if got := failed.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||
}
|
||||
}
|
||||
assertStageRecordsUnchanged(t, failed, upstream)
|
||||
|
||||
retryRunner := &analyzeCheckpointRunner{}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: retryRunner},
|
||||
}); err != nil {
|
||||
t.Fatalf("retry executeStages() error = %v", err)
|
||||
}
|
||||
if want := []string{"quest-log-revised", "player-handout-revised"}; !slices.Equal(retryRunner.Calls, want) {
|
||||
t.Fatalf("retry calls = %#v, want %#v", retryRunner.Calls, want)
|
||||
}
|
||||
|
||||
completed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
if completed.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("aggregate analyze = %#v, want succeeded", completed.Stages["analyze"])
|
||||
}
|
||||
for _, key := range []string{"chronicle", "encounters", "player_handout", "quest_log", "unrelated"} {
|
||||
assertAnalyzeArtifactStatus(t, completed.Stages["analyze"], key, manifest.AnalyzeArtifactCurrent)
|
||||
}
|
||||
assertStageRecordsUnchanged(t, completed, upstream)
|
||||
}
|
||||
|
||||
func TestExecuteStagesJoinsAnalyzeFailureWithTerminalSaveFailure(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||
},
|
||||
}
|
||||
artifactErr := errors.New("injected analyze failure")
|
||||
terminalErr := errors.New("injected terminal save failure")
|
||||
store := &failedAnalyzeSaveStore{Store: &manifest.LocalStore{}, Err: terminalErr}
|
||||
runner := &analyzeCheckpointRunner{FailPrompt: "chronicle", Err: artifactErr}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||
})
|
||||
if !errors.Is(err, artifactErr) || !errors.Is(err, terminalErr) {
|
||||
t.Fatalf("executeStages() error = %v, want analyze and terminal save failures", err)
|
||||
}
|
||||
if !store.Failed {
|
||||
t.Fatal("terminal save failure was not injected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesPersistsAnalyzeCheckpointFailureRecovery(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||
"encounters": {Enabled: true, PromptID: "encounters", OutputPath: "artifacts/encounters.md"},
|
||||
"quest_log": {Enabled: true, PromptID: "quest-log", OutputPath: "artifacts/quest_log.md"},
|
||||
},
|
||||
}
|
||||
baseStore := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||
seed.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
checkpointErr := errors.New("injected artifact checkpoint failure")
|
||||
store := ¤tAnalyzeCheckpointFailingStore{
|
||||
Store: baseStore,
|
||||
CurrentLimit: 2,
|
||||
Err: checkpointErr,
|
||||
}
|
||||
runner := &analyzeCheckpointRunner{}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||
})
|
||||
if !errors.Is(err, checkpointErr) {
|
||||
t.Fatalf("executeStages() error = %v, want checkpoint failure", err)
|
||||
}
|
||||
if want := []string{"chronicle", "encounters"}; !slices.Equal(runner.Calls, want) {
|
||||
t.Fatalf("adapter calls = %#v, want %#v", runner.Calls, want)
|
||||
}
|
||||
|
||||
persisted := loadAnalyzeRecoveryManifest(t, baseStore, cfg)
|
||||
if persisted.Stages["analyze"].Status != manifest.StatusFailed {
|
||||
t.Fatalf("aggregate analyze = %#v, want failed", persisted.Stages["analyze"])
|
||||
}
|
||||
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "encounters", manifest.AnalyzeArtifactFailed)
|
||||
if _, exists := persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"]; exists {
|
||||
t.Fatalf("unattempted quest log appeared: %#v", persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"])
|
||||
}
|
||||
for _, name := range []string{"publish", "notify"} {
|
||||
if got := persisted.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type analyzeCheckpointRunner struct {
|
||||
Calls []string
|
||||
FailPrompt string
|
||||
Err error
|
||||
}
|
||||
|
||||
type failedAnalyzeSaveStore struct {
|
||||
manifest.Store
|
||||
Err error
|
||||
Failed bool
|
||||
}
|
||||
|
||||
type currentAnalyzeCheckpointFailingStore struct {
|
||||
manifest.Store
|
||||
CurrentLimit int
|
||||
Err error
|
||||
Failed bool
|
||||
}
|
||||
|
||||
func (s *currentAnalyzeCheckpointFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if !s.Failed {
|
||||
current := 0
|
||||
if record := m.Stages["analyze"]; record != nil && record.Status == manifest.StatusRunning {
|
||||
for _, artifact := range record.AnalyzeArtifacts {
|
||||
if artifact.Status == manifest.AnalyzeArtifactCurrent {
|
||||
current++
|
||||
}
|
||||
}
|
||||
}
|
||||
if current >= s.CurrentLimit {
|
||||
s.Failed = true
|
||||
return s.Err
|
||||
}
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func (s *failedAnalyzeSaveStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if record := m.Stages["analyze"]; !s.Failed && record != nil && record.Status == manifest.StatusFailed {
|
||||
s.Failed = true
|
||||
return s.Err
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func (r *analyzeCheckpointRunner) RunArtifact(
|
||||
ctx context.Context,
|
||||
req scriptorium.RunArtifactRequest,
|
||||
) (scriptorium.ArtifactResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
r.Calls = append(r.Calls, req.PromptID)
|
||||
if req.PromptID == r.FailPrompt {
|
||||
return scriptorium.ArtifactResult{}, r.Err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(req.OutputPath), 0o755); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
if err := os.WriteFile(req.OutputPath, []byte("generated "+req.PromptID+"\n"), 0o644); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
return scriptorium.ArtifactResult{
|
||||
OutputPath: req.OutputPath,
|
||||
CommandMode: scriptorium.CommandModeRun,
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: req.ProfileID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (*analyzeCheckpointRunner) RenderArtifact(
|
||||
context.Context,
|
||||
scriptorium.RenderArtifactRequest,
|
||||
) (scriptorium.ArtifactResult, error) {
|
||||
return scriptorium.ArtifactResult{}, errors.New("unexpected render")
|
||||
}
|
||||
|
||||
func checkpointRecoveryArtifacts(revision string) map[string]config.ScriptoriumArtifactConfig {
|
||||
return map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {
|
||||
Enabled: true, PromptID: "chronicle-" + revision, OutputPath: "artifacts/chronicle.md",
|
||||
},
|
||||
"encounters": {
|
||||
Enabled: true, DependsOn: []string{"chronicle"}, PromptID: "encounters-" + revision,
|
||||
OutputPath: "artifacts/encounters.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"chronicle": {Source: "narratio.artifact.chronicle", Required: true},
|
||||
},
|
||||
},
|
||||
"quest_log": {
|
||||
Enabled: true, DependsOn: []string{"encounters"}, PromptID: "quest-log-" + revision,
|
||||
OutputPath: "artifacts/quest_log.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"encounters": {Source: "narratio.artifact.encounters", Required: true},
|
||||
},
|
||||
},
|
||||
"player_handout": {
|
||||
Enabled: true, DependsOn: []string{"quest_log"}, PromptID: "player-handout-" + revision,
|
||||
OutputPath: "artifacts/player_handout.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"quest_log": {Source: "narratio.artifact.quest_log", Required: true},
|
||||
},
|
||||
},
|
||||
"unrelated": {
|
||||
Enabled: true, PromptID: "unrelated", OutputPath: "artifacts/unrelated.md",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustSelectStage(t *testing.T, name string) stage.Stage {
|
||||
t.Helper()
|
||||
selected, err := stage.Select(name)
|
||||
if err != nil {
|
||||
t.Fatalf("Select(%q) error = %v", name, err)
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func loadAnalyzeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
m, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("load session manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func assertAnalyzeArtifactStatus(
|
||||
t *testing.T,
|
||||
record *manifest.StageRecord,
|
||||
key string,
|
||||
want manifest.AnalyzeArtifactStatus,
|
||||
) {
|
||||
t.Helper()
|
||||
got, exists := record.AnalyzeArtifacts[key]
|
||||
if !exists || got.Status != want {
|
||||
t.Fatalf("artifact %q = %#v, want status %q", key, got, want)
|
||||
}
|
||||
if want != manifest.AnalyzeArtifactCurrent && got.Output != nil {
|
||||
t.Fatalf("artifact %q retained output authority: %#v", key, got)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneStageRecords(m *manifest.Manifest, names ...string) map[string]manifest.StageRecord {
|
||||
clones := make(map[string]manifest.StageRecord, len(names))
|
||||
for _, name := range names {
|
||||
clones[name] = *m.Stages[name]
|
||||
}
|
||||
return clones
|
||||
}
|
||||
|
||||
func assertStageRecordsUnchanged(t *testing.T, m *manifest.Manifest, want map[string]manifest.StageRecord) {
|
||||
t.Helper()
|
||||
for name, expected := range want {
|
||||
got := m.Stages[name]
|
||||
if got == nil {
|
||||
t.Fatalf("upstream stage %q is missing", name)
|
||||
}
|
||||
if diff := cmp.Diff(expected, *got); diff != "" {
|
||||
t.Fatalf("upstream stage %q changed (-want +got):\n%s", name, diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type validatedAnalyzeProjection struct {
|
||||
session map[string]manifest.AnalyzeArtifactRecord
|
||||
invocation map[string]manifest.AnalyzeArtifactRecord
|
||||
}
|
||||
|
||||
type analyzeStateSnapshot struct {
|
||||
version int
|
||||
records map[string]manifest.AnalyzeArtifactRecord
|
||||
}
|
||||
|
||||
func captureAnalyzeState(manifestValue *manifest.Manifest, stageName string) analyzeStateSnapshot {
|
||||
if manifestValue == nil || stageName != "analyze" || manifestValue.Stages["analyze"] == nil {
|
||||
return analyzeStateSnapshot{}
|
||||
}
|
||||
record := manifestValue.Stages["analyze"]
|
||||
return analyzeStateSnapshot{
|
||||
version: record.AnalyzeStateVersion,
|
||||
records: manifest.CloneAnalyzeArtifactCollection(record.AnalyzeArtifacts),
|
||||
}
|
||||
}
|
||||
|
||||
func restoreAnalyzeState(manifestValue *manifest.Manifest, snapshot analyzeStateSnapshot) {
|
||||
if manifestValue == nil || manifestValue.Stages["analyze"] == nil {
|
||||
return
|
||||
}
|
||||
record := manifestValue.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = snapshot.version
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(snapshot.records)
|
||||
}
|
||||
|
||||
func validateSuccessfulAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
||||
if result == nil || result.AnalyzeState == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if stageName != "analyze" {
|
||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection", stageName)
|
||||
}
|
||||
if result.Disposition == stage.StageDispositionSkipped {
|
||||
return nil, fmt.Errorf("skipped analyze result cannot contain analyze-owned state projection")
|
||||
}
|
||||
if len(result.Outputs) != 0 {
|
||||
return nil, fmt.Errorf("analyze result with state projection cannot contain ordinary outputs")
|
||||
}
|
||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
||||
}
|
||||
|
||||
func validateFailedAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
||||
if result == nil || result.AnalyzeState == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if stageName != "analyze" {
|
||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection with an error", stageName)
|
||||
}
|
||||
if result.Disposition != stage.StageDispositionSucceeded || result.SkipReason != "" || len(result.Outputs) != 0 || len(result.Logs) != 0 || len(result.GeneratedConfigs) != 0 || len(result.Metadata) != 0 {
|
||||
return nil, fmt.Errorf("analyze result with an error may contain only analyze-owned state projection")
|
||||
}
|
||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
||||
}
|
||||
|
||||
func validateAndCloneAnalyzeProjection(projection *stage.AnalyzeStateProjection) (*validatedAnalyzeProjection, error) {
|
||||
session := manifest.CloneAnalyzeArtifactCollection(projection.Session)
|
||||
invocation := manifest.CloneAnalyzeArtifactCollection(projection.Invocation)
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, session); err != nil {
|
||||
return nil, fmt.Errorf("validate reconciled session analyze state: %w", err)
|
||||
}
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, invocation); err != nil {
|
||||
return nil, fmt.Errorf("validate invocation analyze state: %w", err)
|
||||
}
|
||||
for key, invocationRecord := range invocation {
|
||||
sessionRecord, ok := session[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invocation analyze artifact %q is absent from reconciled session state", key)
|
||||
}
|
||||
if !reflect.DeepEqual(invocationRecord, sessionRecord) {
|
||||
return nil, fmt.Errorf("invocation analyze artifact %q contradicts reconciled session state", key)
|
||||
}
|
||||
}
|
||||
return &validatedAnalyzeProjection{session: session, invocation: invocation}, nil
|
||||
}
|
||||
|
||||
func applyAnalyzeProjection(
|
||||
sessionManifest *manifest.Manifest,
|
||||
runManifest *manifest.RunManifest,
|
||||
projection *validatedAnalyzeProjection,
|
||||
) {
|
||||
if projection == nil {
|
||||
return
|
||||
}
|
||||
if sessionManifest != nil && sessionManifest.Stages["analyze"] != nil {
|
||||
record := sessionManifest.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.session)
|
||||
}
|
||||
if runManifest != nil && runManifest.Stages["analyze"] != nil {
|
||||
record := runManifest.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.invocation)
|
||||
}
|
||||
}
|
||||
|
||||
func analyzeProjectionOutputs(records map[string]manifest.AnalyzeArtifactRecord, producerRunID string) []manifest.ArtifactRecord {
|
||||
keys := make([]string, 0, len(records))
|
||||
for key, record := range records {
|
||||
if record.Status != manifest.AnalyzeArtifactCurrent || record.Output == nil {
|
||||
continue
|
||||
}
|
||||
if producerRunID != "" && record.ProducerRunID != producerRunID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
outputs := make([]manifest.ArtifactRecord, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
record := manifest.CloneAnalyzeArtifactCollection(map[string]manifest.AnalyzeArtifactRecord{key: records[key]})[key]
|
||||
output := *record.Output
|
||||
if output.ProducerRunID == "" {
|
||||
output.ProducerRunID = record.ProducerRunID
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
@@ -1,346 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type projectionStage struct {
|
||||
name string
|
||||
run func(*stage.Env, *manifest.Manifest) (*stage.StageResult, error)
|
||||
}
|
||||
|
||||
func (s projectionStage) Name() string { return s.name }
|
||||
func (s projectionStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
return s.run(env, m)
|
||||
}
|
||||
|
||||
func TestExecuteStagesProjectsSeparateSessionAndInvocationAnalyzeState(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
oldAt := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
oldRecord := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", oldAt)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
staleRecord := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactStale, m.RunID, time.Now().UTC())
|
||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": oldRecord,
|
||||
"quest_log": staleRecord,
|
||||
"session_recap": newRecord,
|
||||
}
|
||||
return &stage.StageResult{
|
||||
Logs: []string{"aggregate-analyze.log"},
|
||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: session,
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": oldRecord,
|
||||
"session_recap": newRecord,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(session) error = %v", err)
|
||||
}
|
||||
analyze := sessionManifest.Stages["analyze"]
|
||||
if analyze.AnalyzeStateVersion != manifest.AnalyzeStateContractVersion || len(analyze.AnalyzeArtifacts) != 3 {
|
||||
t.Fatalf("session analyze state = %#v", analyze)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(analyze.Outputs); !reflect.DeepEqual(got, []string{"player_handout", "session_recap"}) {
|
||||
t.Fatalf("session aggregate outputs = %#v, want current records only", got)
|
||||
}
|
||||
if len(analyze.Logs) != 1 || analyze.Logs[0] != "aggregate-analyze.log" {
|
||||
t.Fatalf("session aggregate logs = %#v", analyze.Logs)
|
||||
}
|
||||
|
||||
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun() error = %v", err)
|
||||
}
|
||||
runAnalyze := runManifest.Stages["analyze"]
|
||||
if len(runAnalyze.AnalyzeArtifacts) != 2 || runAnalyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent {
|
||||
t.Fatalf("invocation analyze state = %#v", runAnalyze.AnalyzeArtifacts)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(runAnalyze.Outputs); !reflect.DeepEqual(got, []string{"session_recap"}) {
|
||||
t.Fatalf("invocation outputs = %#v, want produced artifact only", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesPersistsRestrictedAnalyzeStateOnPartialError(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now),
|
||||
}
|
||||
seed.MarkStageSucceeded("publish", now, nil)
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
unrelated := seed.Stages["analyze"].AnalyzeArtifacts["player_handout"]
|
||||
completed := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
failed := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactFailed, m.RunID, time.Now().UTC())
|
||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": unrelated,
|
||||
"quest_log": failed,
|
||||
"session_recap": completed,
|
||||
}
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: session,
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"quest_log": failed,
|
||||
"session_recap": completed,
|
||||
},
|
||||
}}, errors.New("quest log failed")
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "quest log failed") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load(session) error = %v", err)
|
||||
}
|
||||
analyze := loaded.Stages["analyze"]
|
||||
if analyze.Status != manifest.StatusFailed || len(analyze.Outputs) != 0 {
|
||||
t.Fatalf("aggregate analyze state = %#v, want failed without outputs", analyze)
|
||||
}
|
||||
if analyze.AnalyzeArtifacts["player_handout"].Status != manifest.AnalyzeArtifactCurrent ||
|
||||
analyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent ||
|
||||
analyze.AnalyzeArtifacts["quest_log"].Status != manifest.AnalyzeArtifactFailed {
|
||||
t.Fatalf("partial session projection = %#v", analyze.AnalyzeArtifacts)
|
||||
}
|
||||
if loaded.Stages["publish"].Status != manifest.StatusStale {
|
||||
t.Fatalf("publish status = %q, want stale", loaded.Stages["publish"].Status)
|
||||
}
|
||||
|
||||
runsDir := artifacts.SessionRunsDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
entries, err := os.ReadDir(runsDir)
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("run directory entries = %#v, error = %v", entries, err)
|
||||
}
|
||||
runManifest, err := store.LoadRun(context.Background(), filepath.Join(runsDir, entries[0].Name(), "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun() error = %v", err)
|
||||
}
|
||||
runAnalyze := runManifest.Stages["analyze"]
|
||||
if runAnalyze.Status != manifest.StatusFailed || len(runAnalyze.AnalyzeArtifacts) != 2 || len(runAnalyze.Outputs) != 0 {
|
||||
t.Fatalf("partial invocation projection = %#v", runAnalyze)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsInvalidAnalyzeProjectionWithoutReplacingPriorState(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
invalid := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
invalid.Output.Checksum = "invalid"
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
||||
}}, errors.New("analysis failed")
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum") {
|
||||
t.Fatalf("executeStages() error = %v, want projection validation failure", err)
|
||||
}
|
||||
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() error = %v", loadErr)
|
||||
}
|
||||
if len(loaded.Stages["analyze"].AnalyzeArtifacts) != 1 || !reflect.DeepEqual(loaded.Stages["analyze"].AnalyzeArtifacts["player_handout"], prior) {
|
||||
t.Fatalf("prior state replaced by invalid projection: %#v", loaded.Stages["analyze"].AnalyzeArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRollsBackAnalyzeAuthorityWhenProjectionSaveFails(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageReturned := false
|
||||
store := &analyzeProjectionFailingStore{delegate: &manifest.LocalStore{}, shouldFail: func(m *manifest.Manifest) bool {
|
||||
return stageReturned && m.Stages["analyze"] != nil && m.Stages["analyze"].Status == manifest.StatusSucceeded && m.Stages["analyze"].AnalyzeArtifacts["session_recap"].Status == manifest.AnalyzeArtifactCurrent
|
||||
}}
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
stageReturned = true
|
||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": prior,
|
||||
"session_recap": newRecord,
|
||||
},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": newRecord},
|
||||
}}, nil
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{
|
||||
Force: true,
|
||||
Env: &Env{ManifestStore: store},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "injected analyze projection save failure") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if !store.failed {
|
||||
t.Fatal("projection persistence failure was not injected")
|
||||
}
|
||||
loaded, loadErr := store.delegate.Load(context.Background(), manifestPathFor(cfg))
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() error = %v", loadErr)
|
||||
}
|
||||
analyze := loaded.Stages["analyze"]
|
||||
if analyze.Status != manifest.StatusFailed || len(analyze.AnalyzeArtifacts) != 1 || !reflect.DeepEqual(analyze.AnalyzeArtifacts["player_handout"], prior) {
|
||||
t.Fatalf("durable analyze state after rollback = %#v", analyze)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsAnalyzeProjectionFromOtherStage(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
stageToRun := projectionStage{name: "prepare", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
}}, nil
|
||||
}}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "returned analyze-owned state projection") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsContradictoryAnalyzeResultWithError(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{
|
||||
Outputs: []artifacts.Ref{{Kind: "session_recap", RelativePath: "artifacts/session-recap.md"}},
|
||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
},
|
||||
}, errors.New("analysis failed")
|
||||
}}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "may contain only analyze-owned state projection") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesExposesSelectedForceDecisionToStage(t *testing.T) {
|
||||
for _, force := range []bool{false, true} {
|
||||
t.Run(strings.ToLower(strings.TrimSpace(map[bool]string{false: "ordinary", true: "forced"}[force])), func(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
captured := !force
|
||||
stageToRun := projectionStage{name: "prepare", run: func(env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
captured = env.Force
|
||||
return &stage.StageResult{}, nil
|
||||
}}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: force}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if captured != force {
|
||||
t.Fatalf("stage env force = %v, want %v", captured, force)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func appAnalyzeRecord(key string, status manifest.AnalyzeArtifactStatus, producerRunID string, at time.Time) manifest.AnalyzeArtifactRecord {
|
||||
record := manifest.AnalyzeArtifactRecord{
|
||||
Key: key,
|
||||
Status: status,
|
||||
ProducerRunID: producerRunID,
|
||||
UpdatedAt: at,
|
||||
}
|
||||
if status == manifest.AnalyzeArtifactFailed {
|
||||
record.Error = "scriptorium failed"
|
||||
return record
|
||||
}
|
||||
if status != manifest.AnalyzeArtifactCurrent {
|
||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
||||
record.Fingerprint = strings.Repeat("b", 64)
|
||||
return record
|
||||
}
|
||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
||||
record.Fingerprint = strings.Repeat("a", 64)
|
||||
record.OutputSize = 42
|
||||
record.Output = &manifest.ArtifactRecord{
|
||||
Kind: key,
|
||||
SourceID: artifacts.ConfiguredArtifactSourceID(key),
|
||||
LocalPath: "artifacts/" + strings.ReplaceAll(key, "_", "-") + ".md",
|
||||
ProducerRunID: producerRunID,
|
||||
Checksum: strings.Repeat("c", 64),
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: "text/markdown", SchemaID: "narratio." + key, SchemaVersion: "1",
|
||||
},
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func analyzeArtifactOutputKeys(outputs []manifest.ArtifactRecord) []string {
|
||||
keys := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
keys = append(keys, strings.TrimPrefix(output.SourceID, "narratio.artifact."))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
type analyzeProjectionFailingStore struct {
|
||||
delegate *manifest.LocalStore
|
||||
shouldFail func(*manifest.Manifest) bool
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Create(ctx, sessionID)
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Load(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Load(ctx, path)
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if !s.failed && s.shouldFail != nil && s.shouldFail(m) {
|
||||
s.failed = true
|
||||
return errors.New("injected analyze projection save failure")
|
||||
}
|
||||
return s.delegate.Save(ctx, path, m)
|
||||
}
|
||||
43
internal/app/artifact_evidence_test.go
Normal file
43
internal/app/artifact_evidence_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func setCurrentBuiltInEvidence(t *testing.T, m *manifest.Manifest, stage, kind, sourceID, path string) {
|
||||
t.Helper()
|
||||
checksum, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.MarkStageSucceeded(stage, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||
Kind: kind, SourceID: sourceID, LocalPath: path, ProducerRunID: stage + "-run", Checksum: checksum,
|
||||
}})
|
||||
}
|
||||
|
||||
func seedCurrentBuiltInEvidence(t *testing.T, manifestPath, sessionID, campaign, stage, kind, sourceID, path string) {
|
||||
t.Helper()
|
||||
m := manifest.New(sessionID, time.Now().UTC())
|
||||
m.Campaign = campaign
|
||||
setCurrentBuiltInEvidence(t, m, stage, kind, sourceID, path)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setCurrentPreviousArtifactInput(t *testing.T, m *manifest.Manifest, path string) {
|
||||
t.Helper()
|
||||
checksum, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Inputs = append(m.Inputs, manifest.InputRecord{
|
||||
Kind: "previous_artifact", Source: artifactpolicy.PreviousSessionSourceID("session_recap"), Path: path, Checksum: checksum,
|
||||
})
|
||||
}
|
||||
@@ -19,11 +19,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestAssembledFullRunUsesCanonicalOrderAndBoundedRunManifests(t *testing.T) {
|
||||
func TestAssembledFullRunUsesCanonicalOrder(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
canonical := []string{
|
||||
"prepare", "transcribe", "merge", "polish", "normalize", "trim",
|
||||
"render", "extract", "analyze", "publish", "notify",
|
||||
"render", "extract", "analyze", "publish", "notify", "cleanup",
|
||||
}
|
||||
var order []string
|
||||
stages := make([]stage.Stage, 0, len(canonical))
|
||||
@@ -37,13 +37,6 @@ func TestAssembledFullRunUsesCanonicalOrderAndBoundedRunManifests(t *testing.T)
|
||||
if !reflect.DeepEqual(order, canonical) || !reflect.DeepEqual(summary.Executed, canonical) {
|
||||
t.Fatalf("execution order=%#v summary=%#v, want %#v", order, summary.Executed, canonical)
|
||||
}
|
||||
runManifest, err := (&manifest.LocalStore{}).LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runManifest.RequestedStages, canonical) {
|
||||
t.Fatalf("requested stages = %#v, want canonical order", runManifest.RequestedStages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.T) {
|
||||
@@ -55,9 +48,7 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
force bool
|
||||
}
|
||||
var captured []capturedRequest
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
captured = append(captured, capturedRequest{
|
||||
stages: plan.Names(), artifacts: append([]string(nil), options.SelectedArtifacts...), force: options.Force,
|
||||
})
|
||||
@@ -67,11 +58,12 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
"2026-05-03", "--force", "--from", "extract", "--through", "analyze",
|
||||
"--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}
|
||||
if err := Run(context.Background(), base, &bytes.Buffer{}); err != nil {
|
||||
deps := fakeCommandDependencies(&storage.FakeBackend{}, nil)
|
||||
if err := runWithDependencies(context.Background(), base, &bytes.Buffer{}, executor, deps); err != nil {
|
||||
t.Fatalf("canonical unselected Run() error = %v", err)
|
||||
}
|
||||
selected := append(append([]string(nil), base...), "--artifacts", "session_recap")
|
||||
if err := Run(context.Background(), selected, &bytes.Buffer{}); err != nil {
|
||||
if err := runWithDependencies(context.Background(), selected, &bytes.Buffer{}, executor, deps); err != nil {
|
||||
t.Fatalf("canonical selected Run() error = %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -79,7 +71,7 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
"regenerate-artifacts", "2026-05-03", "--artifacts", "session_recap",
|
||||
"--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}
|
||||
if code := Execute(alias, &stdout, &stderr); code != 0 {
|
||||
if code := executeWithDependencies(alias, &stdout, &stderr, deps, executor); code != 0 {
|
||||
t.Fatalf("alias exit=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(captured) != 3 {
|
||||
@@ -103,7 +95,7 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
plan := mustBoundedPlan(t, selected, selected)
|
||||
runs := 0
|
||||
plan.stages = []stage.Stage{countingStage{name: selected, runs: &runs}}
|
||||
summary, err := executePlan(context.Background(), cfg, plan, RunOptions{Force: true})
|
||||
_, err := executePlan(context.Background(), cfg, plan, RunOptions{Force: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -115,18 +107,11 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
if loaded.Stages[sibling].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("%s sibling = %#v, want succeeded", sibling, loaded.Stages[sibling])
|
||||
}
|
||||
for _, dependent := range []string{"analyze", "publish", "notify"} {
|
||||
for _, dependent := range []string{"analyze", "publish", "notify", "cleanup"} {
|
||||
if loaded.Stages[dependent].Status != manifest.StatusStale {
|
||||
t.Fatalf("%s status = %q, want stale", dependent, loaded.Stages[dependent].Status)
|
||||
}
|
||||
}
|
||||
runManifest, err := (&manifest.LocalStore{}).LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runManifest.RequestedStages, []string{selected}) || len(runManifest.Stages) != 1 {
|
||||
t.Fatalf("bounded run manifest = %#v", runManifest)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -143,7 +128,7 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
if loaded.Stages["extract"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("extract sibling = %#v", loaded.Stages["extract"])
|
||||
}
|
||||
for _, outside := range []string{"analyze", "publish", "notify"} {
|
||||
for _, outside := range []string{"analyze", "publish", "notify", "cleanup"} {
|
||||
if loaded.Stages[outside].Status != manifest.StatusStale {
|
||||
t.Fatalf("outside stage %s = %#v, want stale and unexecuted", outside, loaded.Stages[outside])
|
||||
}
|
||||
@@ -231,20 +216,13 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
t.Fatalf("legacy unselected handout catalog entry = %#v, present=%v", entry, ok)
|
||||
}
|
||||
|
||||
full, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{Env: &Env{Scriptorium: fake}})
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{Env: &Env{Scriptorium: fake}})
|
||||
if err != nil {
|
||||
t.Fatalf("full regeneration: %v", err)
|
||||
}
|
||||
if len(fake.RunRequests) != 2 || fake.RunRequests[1].PromptID != "dnd.player_handout" {
|
||||
t.Fatalf("full requests = %#v, want only missing handout added", fake.RunRequests)
|
||||
}
|
||||
fullRun, err := (&manifest.LocalStore{}).LoadRun(context.Background(), full.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(fullRun.Stages["analyze"].Outputs); !reflect.DeepEqual(got, []string{"player_handout"}) {
|
||||
t.Fatalf("full invocation outputs = %#v, want newly generated handout only", got)
|
||||
}
|
||||
afterFull := loadAssembledManifest(t, cfg)
|
||||
for _, key := range []string{"player_handout", "session_recap"} {
|
||||
if afterFull.Stages["analyze"].AnalyzeArtifacts[key].Status != manifest.AnalyzeArtifactCurrent {
|
||||
@@ -257,7 +235,7 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
t.Fatal(err)
|
||||
}
|
||||
remote := &storage.FakeBackend{}
|
||||
published, err := executeStages(context.Background(), cfg, []stage.Stage{publish}, RunOptions{Env: &Env{ObjectStore: remote}})
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publish}, RunOptions{Env: &Env{ObjectStore: remote}})
|
||||
if err != nil {
|
||||
t.Fatalf("publish current records: %v", err)
|
||||
}
|
||||
@@ -265,13 +243,6 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
if got := afterPublish.Stages["publish"].Metadata["published_files_uploaded"]; got != float64(2) {
|
||||
t.Fatalf("published files = %#v, want 2", got)
|
||||
}
|
||||
publishRun, err := (&manifest.LocalStore{}).LoadRun(context.Background(), published.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(publishRun.RequestedStages, []string{"publish"}) || publishRun.Stages["analyze"] != nil {
|
||||
t.Fatalf("publish run manifest = %#v", publishRun)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testing.T) {
|
||||
@@ -283,10 +254,10 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
|
||||
var validated bytes.Buffer
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
|
||||
t.Fatalf("validate production default: %v", err)
|
||||
t.Fatalf("validate production base: %v", err)
|
||||
}
|
||||
if !strings.Contains(validated.String(), "profile=production") {
|
||||
t.Fatalf("validate output = %q, want production profile", validated.String())
|
||||
if !strings.Contains(validated.String(), "profile=none") {
|
||||
t.Fatalf("validate output = %q, want unprofiled production base", validated.String())
|
||||
}
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigShow, ConfigSources} {
|
||||
if err := command(context.Background(), []string{
|
||||
@@ -297,7 +268,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{
|
||||
"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
||||
"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
||||
}, &diff); err != nil {
|
||||
t.Fatalf("compare profiles: %v", err)
|
||||
}
|
||||
@@ -317,12 +288,10 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
t.Fatalf("plan output = %q, want testing provenance", planned.String())
|
||||
}
|
||||
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
var capturedCfg *config.Config
|
||||
var capturedPlan BoundedPlan
|
||||
var capturedOptions RunOptions
|
||||
executeStagesFn = func(_ context.Context, cfg *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, cfg *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
capturedCfg = cfg
|
||||
capturedPlan = plan
|
||||
capturedOptions = options
|
||||
@@ -330,7 +299,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
runArgs := append(append([]string(nil), common...), "--from", "analyze", "--through", "analyze", "--artifacts", "character_items_arannis")
|
||||
var runOut bytes.Buffer
|
||||
if err := Run(context.Background(), runArgs, &runOut); err != nil {
|
||||
if err := runWithPipelineExecutor(context.Background(), runArgs, &runOut, executor); err != nil {
|
||||
t.Fatalf("bounded family run: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(capturedPlan.Names(), []string{"analyze"}) {
|
||||
@@ -396,7 +365,7 @@ func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *tes
|
||||
if !reflect.DeepEqual(productionRuns, make([]int, len(names))) {
|
||||
t.Fatalf("production runs = %v, want complete reuse", productionRuns)
|
||||
}
|
||||
assertAssembledProvenance(t, store, productionSummary, "production", config.EffectivePipelineDigest(production.Pipeline))
|
||||
assertAssembledProvenance(t, store, productionSummary, "", config.EffectivePipelineDigest(production.Pipeline))
|
||||
|
||||
testingRuns := make([]int, len(names))
|
||||
testingStages := assembledSemanticStages(names, providers, testingRuns)
|
||||
@@ -445,14 +414,11 @@ func assertAssembledProvenance(t *testing.T, store *manifest.LocalStore, summary
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, value := range []*manifest.EffectiveConfigProvenance{session.EffectiveConfig, run.EffectiveConfig} {
|
||||
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
|
||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||
}
|
||||
value := session.EffectiveConfig
|
||||
profileMatches := value != nil && ((profile == "" && value.SelectedProfile == nil) ||
|
||||
(profile != "" && value.SelectedProfile != nil && value.SelectedProfile.Name == profile))
|
||||
if !profileMatches || value.EffectiveConfigDigest != digest {
|
||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -140,9 +140,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
var capturedStages []string
|
||||
var capturedPlan BoundedPlan
|
||||
var capturedOptions RunOptions
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
capturedStages = plan.Names()
|
||||
capturedPlan = plan
|
||||
capturedOptions = options
|
||||
@@ -150,7 +148,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{
|
||||
err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03",
|
||||
"--from", "render",
|
||||
"--through", "extract",
|
||||
@@ -158,7 +156,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
}, &out)
|
||||
}, &out, executor)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,11 @@ import (
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
// state unless cache cleanup is explicitly requested.
|
||||
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
return cleanWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func cleanWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -31,16 +36,16 @@ func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
return err
|
||||
}
|
||||
if all {
|
||||
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||
return cleanAllLocalWithDependencies(flags, dryRun, clearCache, out, deps)
|
||||
}
|
||||
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||
return cleanSessionWithDependencies(ctx, flags, dryRun, clearCache, out, deps)
|
||||
}
|
||||
|
||||
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
func cleanSessionWithDependencies(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer, deps commandDependencies) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
@@ -71,7 +76,7 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||
if err := cleanSessionAudioCacheWithDependencies(ctx, cfg, dryRun, out, deps); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
@@ -80,7 +85,7 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
func cleanAllLocalWithDependencies(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer, deps commandDependencies) error {
|
||||
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
||||
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||
@@ -88,7 +93,7 @@ func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writ
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||
}
|
||||
_, pipelineCfg, err := loadPipelineConfig(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()})
|
||||
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
@@ -194,7 +199,7 @@ func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||
return rootAbs, out, nil
|
||||
}
|
||||
|
||||
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||
func cleanSessionAudioCacheWithDependencies(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer, deps commandDependencies) error {
|
||||
if cfg.Session.Inputs.AudioS3 == nil {
|
||||
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||
return nil
|
||||
@@ -202,7 +207,7 @@ func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool
|
||||
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ inputs:
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||
@@ -105,7 +105,7 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
280
internal/app/cleanup_orchestration_test.go
Normal file
280
internal/app/cleanup_orchestration_test.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
const cleanupOrchestrationRunID = "20260905T010203Z-a1b2c3d4"
|
||||
|
||||
type cleanupPublishEvidenceStage struct {
|
||||
runs *int
|
||||
skipSpoolTarget bool
|
||||
}
|
||||
|
||||
func (cleanupPublishEvidenceStage) Name() string { return "publish" }
|
||||
|
||||
func (s cleanupPublishEvidenceStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
if s.runs != nil {
|
||||
*s.runs++
|
||||
}
|
||||
if !s.skipSpoolTarget {
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, m.Campaign, m.SessionID, m.RunID)
|
||||
if err := os.MkdirAll(spoolTarget, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(spoolTarget, "audio.flac"), []byte("audio"), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, m.Campaign, m.SessionID, m.RunID)
|
||||
if err := os.MkdirAll(workspaceTarget, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stage.StageResult{Metadata: cleanupPublishMetadata(m)}, nil
|
||||
}
|
||||
|
||||
func cleanupPublishMetadata(m *manifest.Manifest) map[string]any {
|
||||
return map[string]any{
|
||||
"uploaded": true,
|
||||
"published_run_id": m.RunID,
|
||||
"remote_commit_key": artifacts.S3RunCommitKey(m.S3SessionPrefix, m.RunID),
|
||||
"current_commit_pointer_key": artifacts.S3CurrentCommitPointerKey(m.S3SessionPrefix),
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupStopsAtPublishAndNotifyFailureBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stages func(*int) []stage.Stage
|
||||
}{
|
||||
{
|
||||
name: "publish failure",
|
||||
stages: func(_ *int) []stage.Stage {
|
||||
return []stage.Stage{failingStage{name: "publish", err: errors.New("publish failed")}, selectedCleanupStage(t)}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "notify failure",
|
||||
stages: func(publishRuns *int) []stage.Stage {
|
||||
return []stage.Stage{
|
||||
cleanupPublishEvidenceStage{runs: publishRuns},
|
||||
failingStage{name: "notify", err: errors.New("notify failed")},
|
||||
selectedCleanupStage(t),
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, remote := cleanupOrchestrationConfig(t)
|
||||
publishRuns := 0
|
||||
_, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, BoundedPlan{stages: test.stages(&publishRuns)},
|
||||
RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("execution error = nil")
|
||||
}
|
||||
loaded := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if loaded.Stages["cleanup"] != nil {
|
||||
t.Fatalf("cleanup record = %#v, want no execution", loaded.Stages["cleanup"])
|
||||
}
|
||||
if test.name == "notify failure" {
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
if _, statErr := os.Stat(workspaceTarget); statErr != nil {
|
||||
t.Fatalf("workspace target removed after notify failure: %v", statErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupFailurePreservesPublishAndRetriesWithoutRepublishing(t *testing.T) {
|
||||
cfg, remote := cleanupOrchestrationConfig(t)
|
||||
badSpoolRoot := filepath.Join(t.TempDir(), "spool-file")
|
||||
if err := os.WriteFile(badSpoolRoot, []byte("not a directory"), 0o644); err != nil {
|
||||
t.Fatalf("write invalid spool root: %v", err)
|
||||
}
|
||||
cfg.Pipeline.Spool.Root = badSpoolRoot
|
||||
remote.SeedObject(storage.FakeObject{Key: "sentinel/remote-state", Data: []byte("unchanged")})
|
||||
publishRuns := 0
|
||||
plan := BoundedPlan{stages: []stage.Stage{
|
||||
cleanupPublishEvidenceStage{runs: &publishRuns, skipSpoolTarget: true},
|
||||
resultStage{name: "notify"},
|
||||
selectedCleanupStage(t),
|
||||
}}
|
||||
|
||||
_, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, plan, RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("cleanup failure = nil")
|
||||
}
|
||||
loaded := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if loaded.Stages["publish"] == nil || loaded.Stages["publish"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("publish record = %#v, want succeeded", loaded.Stages["publish"])
|
||||
}
|
||||
if loaded.Stages["cleanup"] == nil || loaded.Stages["cleanup"].Status != manifest.StatusFailed {
|
||||
t.Fatalf("cleanup record = %#v, want failed", loaded.Stages["cleanup"])
|
||||
}
|
||||
if got := string(remote.Objects["sentinel/remote-state"].Data); got != "unchanged" {
|
||||
t.Fatalf("remote sentinel = %q, want unchanged", got)
|
||||
}
|
||||
if len(remote.Uploads)+len(remote.Downloads)+len(remote.Reads)+len(remote.Deletes)+len(remote.ExistsKeys) != 0 {
|
||||
t.Fatalf("cleanup touched remote storage: %#v", remote)
|
||||
}
|
||||
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
cfg.Pipeline.Spool.Root = t.TempDir()
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
if err := os.MkdirAll(spoolTarget, 0o755); err != nil {
|
||||
t.Fatalf("create retry spool target: %v", err)
|
||||
}
|
||||
summary, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, plan, RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("retry cleanup: %v", err)
|
||||
}
|
||||
if publishRuns != 1 {
|
||||
t.Fatalf("publish runs = %d, want one", publishRuns)
|
||||
}
|
||||
if !reflect.DeepEqual(summary.Executed, []string{"cleanup"}) || !reflect.DeepEqual(summary.Skipped, []string{"publish", "notify"}) {
|
||||
t.Fatalf("retry summary = %#v", summary)
|
||||
}
|
||||
if _, statErr := os.Lstat(workspaceTarget); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("workspace target stat error = %v, want absent", statErr)
|
||||
}
|
||||
if _, statErr := os.Lstat(spoolTarget); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("spool target stat error = %v, want absent", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageCleanupUsesPublishEvidenceAndFailsClosedWithoutIt(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
enableCleanupForCommandTest(t, pipelinePath)
|
||||
cfg, err := config.Load(pipelinePath, campaignPath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load cleanup config: %v", err)
|
||||
}
|
||||
|
||||
publishedRunID := "20260904T010203Z-a1b2c3d4"
|
||||
m := manifest.New(cfg.Session.SessionID, time.Date(2026, 9, 4, 1, 2, 3, 0, time.UTC))
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
m.RunID = publishedRunID
|
||||
m.S3SessionPrefix = artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, m.Campaign, m.SessionID)
|
||||
m.MarkStageSucceeded("publish", time.Date(2026, 9, 4, 1, 3, 0, 0, time.UTC), nil)
|
||||
m.Stages["publish"].Metadata = cleanupPublishMetadata(m)
|
||||
manifestPath := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatalf("save publication evidence: %v", err)
|
||||
}
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID, publishedRunID)
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, m.Campaign, m.SessionID, publishedRunID)
|
||||
for _, target := range []string{workspaceTarget, spoolTarget} {
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatalf("create cleanup target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
if err := RunStage(context.Background(), []string{
|
||||
"cleanup", cfg.Session.SessionID, "--config", pipelinePath,
|
||||
"--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}, &output); err != nil {
|
||||
t.Fatalf("RunStage(cleanup): %v", err)
|
||||
}
|
||||
if !strings.Contains(output.String(), "stage=cleanup executed=1 skipped=0") {
|
||||
t.Fatalf("output = %q, want direct cleanup execution", output.String())
|
||||
}
|
||||
for _, target := range []string{workspaceTarget, spoolTarget} {
|
||||
if _, statErr := os.Lstat(target); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("cleanup target %q stat error = %v, want absent", target, statErr)
|
||||
}
|
||||
}
|
||||
cleaned := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if cleaned.RunID == publishedRunID {
|
||||
t.Fatalf("cleanup invocation run id = %q, want a new invocation identity", cleaned.RunID)
|
||||
}
|
||||
if _, statErr := os.Lstat(cleaned.LocalWorkDir); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("cleanup invocation work directory stat error = %v, want absent", statErr)
|
||||
}
|
||||
|
||||
missingRoot := t.TempDir()
|
||||
missingPipeline, missingCampaign, missingSession := writeValidConfigFiles(t, missingRoot)
|
||||
enableCleanupForCommandTest(t, missingPipeline)
|
||||
err = RunStage(context.Background(), []string{
|
||||
"cleanup", "2026-05-03", "--config", missingPipeline,
|
||||
"--campaign-file", missingCampaign, "--session", missingSession,
|
||||
}, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "cleanup requires a recorded publish stage") {
|
||||
t.Fatalf("RunStage(cleanup) without evidence error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func enableCleanupForCommandTest(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline config: %v", err)
|
||||
}
|
||||
content := strings.Replace(string(data), "workspace:\n root: ", "workspace:\n cleanup_after_publish: true\n root: ", 1)
|
||||
content = strings.Replace(content, "spool:\n root: ", "spool:\n delete_audio_after_publish: true\n root: ", 1)
|
||||
content = strings.Replace(content, " upload_run: false", " upload_run: true", 1)
|
||||
if err := os.WriteFile(pipelinePath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write cleanup config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupOrchestrationConfig(t *testing.T) (*config.Config, *storage.FakeBackend) {
|
||||
t.Helper()
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Spool.Root = t.TempDir()
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||
cfg.Pipeline.Storage.Backend = config.StorageBackendS3
|
||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "archive", RootPrefix: "dnd"}
|
||||
return cfg, &storage.FakeBackend{}
|
||||
}
|
||||
|
||||
func selectedCleanupStage(t *testing.T) stage.Stage {
|
||||
t.Helper()
|
||||
selected, err := BuildSingleStagePlan("cleanup")
|
||||
if err != nil {
|
||||
t.Fatalf("select cleanup: %v", err)
|
||||
}
|
||||
return selected[0]
|
||||
}
|
||||
|
||||
func cleanupOrchestrationDependencies() invocationDependencies {
|
||||
return invocationDependencies{
|
||||
now: func() time.Time { return time.Date(2026, 9, 5, 1, 2, 3, 0, time.UTC) },
|
||||
newRunID: func() (string, error) { return cleanupOrchestrationRunID, nil },
|
||||
}
|
||||
}
|
||||
|
||||
func loadCleanupOrchestrationManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("load manifest: %v", err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
@@ -7,6 +7,16 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type scopedDir struct {
|
||||
RootAbs string
|
||||
TargetAbs string
|
||||
Exists bool
|
||||
}
|
||||
|
||||
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, true)
|
||||
}
|
||||
|
||||
func validateScopedTarget(root, target, policy string, requireDir bool) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
|
||||
57
internal/app/command_dependencies.go
Normal file
57
internal/app/command_dependencies.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type commandDependencies struct {
|
||||
diagnostics io.Writer
|
||||
resolvePipelineConfigPath func(string) (string, error)
|
||||
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
|
||||
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
|
||||
loadSecretsDir func(string) (*configkit.SecretsReport, error)
|
||||
secretsBootstrap *secretsBootstrap
|
||||
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
|
||||
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
|
||||
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
|
||||
executeRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, *RestorePlan, *RestoreReport, storage.ObjectStore) (*RestoreExecutionResult, error)
|
||||
}
|
||||
|
||||
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
|
||||
if deps.diagnostics == nil {
|
||||
deps.diagnostics = io.Discard
|
||||
}
|
||||
if deps.resolvePipelineConfigPath == nil {
|
||||
deps.resolvePipelineConfigPath = resolvePipelineConfigPath
|
||||
}
|
||||
if deps.loadPipelineConfig == nil {
|
||||
deps.loadPipelineConfig = config.LoadPipelineWithOptions
|
||||
}
|
||||
if deps.downloadObjectToTemp == nil {
|
||||
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
|
||||
}
|
||||
if deps.loadSecretsDir == nil {
|
||||
deps.loadSecretsDir = configkit.LoadSecretsDir
|
||||
}
|
||||
if deps.secretsBootstrap == nil {
|
||||
deps.secretsBootstrap = &secretsBootstrap{}
|
||||
}
|
||||
if deps.newObjectStore == nil {
|
||||
deps.newObjectStore = storage.NewObjectStoreFromConfig
|
||||
}
|
||||
if deps.discoverRemoteCurrentState == nil {
|
||||
deps.discoverRemoteCurrentState = discoverRemoteCurrentState
|
||||
}
|
||||
if deps.buildRestorePlan == nil {
|
||||
deps.buildRestorePlan = buildRestorePlan
|
||||
}
|
||||
if deps.executeRestorePlan == nil {
|
||||
deps.executeRestorePlan = executeRestorePlan
|
||||
}
|
||||
return deps
|
||||
}
|
||||
@@ -9,10 +9,20 @@ import (
|
||||
|
||||
var supportedCommands = []string{"version", "run", "regenerate-artifacts", "run-stage", "analyze", "publish", "clean", "session", "config"}
|
||||
|
||||
var runCommandFn = Run
|
||||
|
||||
// Execute dispatches CLI commands and returns a process exit code.
|
||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
return executeWithDependencies(args, stdout, stderr, commandDependencies{}, executePlan)
|
||||
}
|
||||
|
||||
func executeWithPipelineExecutor(args []string, stdout, stderr io.Writer, executor pipelineExecutor) int {
|
||||
return executeWithDependencies(args, stdout, stderr, commandDependencies{}, executor)
|
||||
}
|
||||
|
||||
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
|
||||
if deps.diagnostics == nil {
|
||||
deps.diagnostics = stderr
|
||||
}
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
printUsage(stderr)
|
||||
return 1
|
||||
@@ -27,21 +37,21 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
case "version":
|
||||
err = Version(cmdArgs, stdout)
|
||||
case "run":
|
||||
err = runCommandFn(ctx, cmdArgs, stdout)
|
||||
err = runWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "regenerate-artifacts":
|
||||
err = RegenerateArtifacts(ctx, cmdArgs, stdout)
|
||||
err = regenerateArtifactsWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "run-stage":
|
||||
err = RunStage(ctx, cmdArgs, stdout)
|
||||
err = runStageWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "analyze":
|
||||
err = Analyze(ctx, cmdArgs, stdout)
|
||||
err = analyzeWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "publish":
|
||||
err = Publish(ctx, cmdArgs, stdout)
|
||||
err = publishWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "session":
|
||||
err = Session(ctx, cmdArgs, stdout)
|
||||
err = sessionWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
case "clean":
|
||||
err = Clean(ctx, cmdArgs, stdout)
|
||||
err = cleanWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
case "config":
|
||||
err = Config(ctx, cmdArgs, stdout)
|
||||
err = configWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||
printUsage(stderr)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
@@ -31,7 +32,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=12 skipped=2; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "analyze: skip\n targets: none\n prerequisites: none\n execute: none\n reuse: none\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
@@ -58,9 +59,10 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
|
||||
func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
controlledDiscovery bool
|
||||
}{
|
||||
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
||||
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
||||
@@ -68,7 +70,7 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
|
||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided", controlledDiscovery: true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -76,7 +78,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
deps := commandDependencies{}
|
||||
executor := pipelineExecutor(executePlan)
|
||||
missingPath := ""
|
||||
if tc.controlledDiscovery {
|
||||
deps, missingPath, executor = guardedMissingPipelineDiscovery(t)
|
||||
}
|
||||
code := executeWithDependencies(tc.args, &stdout, &stderr, deps, executor)
|
||||
if code == 0 {
|
||||
t.Fatalf("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -86,6 +94,11 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want to contain %q", stderr.String(), tc.want)
|
||||
}
|
||||
if tc.controlledDiscovery {
|
||||
if got := stderr.String(); !strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||
t.Fatalf("stderr = %q, want controlled searched path %q", got, missingPath)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -126,7 +139,9 @@ func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
|
||||
polishedPath := filepath.Join(workRoot, "transcripts", "polished.json")
|
||||
mustWriteTestFile(t, polishedPath, `{"segments":[{"id":1}]}`)
|
||||
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), "2026-05-03", "sample-campaign", "polish", artifacts.TranscriptOutputKindPolished, artifacts.ArtifactTranscriptPolished, polishedPath)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -189,7 +204,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
sessionID := "2026-05-03"
|
||||
secretsDir := filepath.Join(configDir, "secrets")
|
||||
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
||||
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||
@@ -208,15 +223,13 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
binary: ` + seriatimBinary + `
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
llm_api_key_env: OPENROUTER_API_KEY
|
||||
llm_api_key: OPENROUTER_API_KEY
|
||||
notification:
|
||||
mode: noop
|
||||
`
|
||||
@@ -227,8 +240,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -249,7 +260,9 @@ inputs:
|
||||
})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
basePath := filepath.Join(workRoot, "transcripts", "base.json")
|
||||
mustWriteTestFile(t, basePath, `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), sessionID, "sample-campaign", "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
@@ -261,9 +274,31 @@ inputs:
|
||||
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||
}
|
||||
foundReference := false
|
||||
if err := filepath.WalkDir(workRoot, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil || entry.IsDir() {
|
||||
return walkErr
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bytes.Contains(data, []byte("from-secret-file")) {
|
||||
t.Fatalf("workspace output %q exposed credential value", path)
|
||||
}
|
||||
if bytes.Contains(data, []byte("OPENROUTER_API_KEY")) {
|
||||
foundReference = true
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("inspect polish outputs: %v", err)
|
||||
}
|
||||
if !foundReference {
|
||||
t.Fatal("polish outputs omitted safe credential reference")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
func TestExecuteRunFailsWhenExplicitSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
@@ -274,8 +309,6 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./missing-secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
@@ -292,8 +325,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -304,11 +335,11 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--secrets-dir", filepath.Join(configDir, "missing-secrets")}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
|
||||
if !strings.Contains(stderr.String(), "bootstrap secrets directory") {
|
||||
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
||||
}
|
||||
}
|
||||
@@ -335,7 +366,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=12 skipped=2; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
@@ -389,14 +420,13 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -522,7 +552,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
|
||||
@@ -542,8 +571,7 @@ inputs:
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
@@ -568,13 +596,12 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(dir, "party.yml"), "legacy: party\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
return campaignPath
|
||||
}
|
||||
|
||||
|
||||
211
internal/app/config_artifacts_test.go
Normal file
211
internal/app/config_artifacts_test.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const artifactCatalogHeader = "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source\n"
|
||||
|
||||
func TestConfigArtifactsReportsOrdinaryCatalogWithoutCampaign(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: `+filepath.Join(dir, "workspace")+`
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
zeta:
|
||||
enabled: false
|
||||
output_path: artifacts/zeta.md
|
||||
alpha:
|
||||
enabled: false
|
||||
output_path: artifacts/alpha.md
|
||||
inputs:
|
||||
upstream: {source: narratio.artifact.zeta, required: true}
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||
t.Fatalf("ConfigArtifacts() error = %v", err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
|
||||
if len(lines) != 3 || !strings.HasPrefix(lines[1], "alpha\t") || !strings.HasPrefix(lines[2], "zeta\t") {
|
||||
t.Fatalf("catalog order/output = %q", out.String())
|
||||
}
|
||||
alpha := strings.Split(lines[1], "\t")
|
||||
if len(alpha) != 10 {
|
||||
t.Fatalf("alpha fields = %#v", alpha)
|
||||
}
|
||||
if alpha[1] != "narratio.artifact.alpha" || alpha[2] != pipelinePath || alpha[3] != "false" || alpha[4] != "artifacts/alpha.md" || alpha[5] != "zeta" {
|
||||
t.Fatalf("alpha record = %#v", alpha)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsEmptyCatalogWritesOnlyHeader(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.String() != artifactCatalogHeader {
|
||||
t.Fatalf("empty catalog = %q, want header only", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsGeneratedFamilyProvenance(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, out.String())
|
||||
meta := records["character_meta_arannis"]
|
||||
note := records["character_note_arannis"]
|
||||
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
|
||||
if len(meta) != 10 || meta[6] != "character_meta" || meta[7] != "arannis" || meta[8] != pipelinePath || meta[9] != partyPath {
|
||||
t.Fatalf("generated meta record = %#v", meta)
|
||||
}
|
||||
if len(note) != 10 || note[5] != "character_meta_arannis" || note[6] != "character_note" {
|
||||
t.Fatalf("generated note record = %#v", note)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsProfiledFamilyContributors(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifact_families:
|
||||
character_note:
|
||||
prompt_id: dnd.character_note.testing
|
||||
`)
|
||||
|
||||
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}
|
||||
var catalog bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), args, &catalog); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, catalog.String())
|
||||
if got := strings.Split(records["character_note_arannis"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||
t.Fatalf("generated family definitions = %#v", got)
|
||||
}
|
||||
|
||||
var sources bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), args, &sources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "scriptorium.artifacts.character_note_arannis.prompt_id\tfamily\t" + profilePath
|
||||
if !strings.Contains(sources.String(), want) {
|
||||
t.Fatalf("sources output missing profiled family contributor %q:\n%s", want, sources.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsBaseAndProfileContributors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
notes:
|
||||
enabled: false
|
||||
output_path: artifacts/notes.md
|
||||
`)
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifacts:
|
||||
notes:
|
||||
enabled: true
|
||||
prompt_id: notes
|
||||
profile_only:
|
||||
enabled: false
|
||||
output_path: artifacts/profile-only.md
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, out.String())
|
||||
if got := strings.Split(records["notes"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||
t.Fatalf("notes definitions = %#v", got)
|
||||
}
|
||||
if got := records["profile_only"][2]; got != profilePath {
|
||||
t.Fatalf("profile-only definition = %q, want %q", got, profilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsIdentityDoesNotDependOnDefinitionFilename(t *testing.T) {
|
||||
inlineDir := t.TempDir()
|
||||
inlinePath := filepath.Join(inlineDir, "pipeline.yml")
|
||||
mustWriteTestFile(t, inlinePath, artifactIdentityPipeline("", "recap"))
|
||||
importDir := t.TempDir()
|
||||
importPath := filepath.Join(importDir, "pipeline.yml")
|
||||
mustWriteTestFile(t, importPath, artifactIdentityPipeline("artifacts.yml", ""))
|
||||
mustWriteTestFile(t, filepath.Join(importDir, "artifacts.yml"), "scriptorium:\n artifacts:\n recap:\n enabled: false\n output_path: artifacts/recap.md\n")
|
||||
|
||||
for _, path := range []string{inlinePath, importPath} {
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", path}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := catalogRecordsByKey(t, out.String())["recap"][1]; got != "narratio.artifact.recap" {
|
||||
t.Fatalf("source identity for %s = %q", path, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsPropagatesWriterErrorsAndEscapesFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
want := errors.New("write rejected")
|
||||
err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, rejectingCatalogWriter{err: want})
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("ConfigArtifacts() error = %v, want writer error", err)
|
||||
}
|
||||
if got := escapeCatalogField("a\\b\tc\nd\r"); got != `a\\b\tc\nd\r` {
|
||||
t.Fatalf("escaped field = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogRecordsByKey(t *testing.T, output string) map[string][]string {
|
||||
t.Helper()
|
||||
lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n")
|
||||
if len(lines) == 0 || lines[0]+"\n" != artifactCatalogHeader {
|
||||
t.Fatalf("catalog header = %q", output)
|
||||
}
|
||||
records := make(map[string][]string, len(lines)-1)
|
||||
for _, line := range lines[1:] {
|
||||
fields := strings.Split(line, "\t")
|
||||
records[fields[0]] = fields
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func artifactIdentityPipeline(importPath, inlineKey string) string {
|
||||
document := "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\nscriptorium:\n binary: scriptorium\n"
|
||||
if importPath != "" {
|
||||
document = "configkit:\n imports:\n - path: " + importPath + "\n" + document
|
||||
}
|
||||
if inlineKey != "" {
|
||||
document += " artifacts:\n " + inlineKey + ":\n enabled: false\n output_path: artifacts/recap.md\n"
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
type rejectingCatalogWriter struct{ err error }
|
||||
|
||||
func (w rejectingCatalogWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
@@ -17,6 +18,7 @@ type inspectionFlags struct {
|
||||
campaignPath string
|
||||
campaignFilePath string
|
||||
profile singletonStringFlag
|
||||
secretsDir singletonStringFlag
|
||||
}
|
||||
|
||||
type inspectionConfig struct {
|
||||
@@ -29,8 +31,13 @@ type inspectionConfig struct {
|
||||
|
||||
// Config dispatches read-only pipeline configuration commands.
|
||||
func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|diff")
|
||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|artifacts|diff")
|
||||
}
|
||||
if args[0] == "--help" || args[0] == "-h" {
|
||||
writeConfigCommandUsage(out)
|
||||
@@ -38,13 +45,15 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return ConfigValidate(ctx, args[1:], out)
|
||||
return configValidateWithDependencies(ctx, args[1:], out, deps)
|
||||
case "show":
|
||||
return ConfigShow(ctx, args[1:], out)
|
||||
return configShowWithDependencies(ctx, args[1:], out, deps)
|
||||
case "sources":
|
||||
return ConfigSources(ctx, args[1:], out)
|
||||
return configSourcesWithDependencies(ctx, args[1:], out, deps)
|
||||
case "artifacts":
|
||||
return configArtifactsWithDependencies(ctx, args[1:], out, deps)
|
||||
case "diff":
|
||||
return ConfigDiff(ctx, args[1:], out)
|
||||
return configDiffWithDependencies(ctx, args[1:], out, deps)
|
||||
default:
|
||||
return fmt.Errorf("config: unknown subcommand %q", args[0])
|
||||
}
|
||||
@@ -52,7 +61,11 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigValidate validates one fully resolved effective pipeline without
|
||||
// loading session state or constructing runtime collaborators.
|
||||
func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configValidateWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configValidateWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config validate", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -60,7 +73,7 @@ func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config validate: %w", err)
|
||||
}
|
||||
@@ -71,7 +84,11 @@ func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigShow validates and renders one fully resolved effective pipeline
|
||||
// without loading session state or constructing runtime collaborators.
|
||||
func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigShow(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configShowWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configShowWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config show", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -79,7 +96,7 @@ func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config show: %w", err)
|
||||
}
|
||||
@@ -93,7 +110,11 @@ func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigSources validates and reports deterministic effective configuration
|
||||
// ownership without loading session state or runtime collaborators.
|
||||
func ConfigSources(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigSources(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configSourcesWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configSourcesWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config sources", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -101,7 +122,7 @@ func ConfigSources(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config sources: %w", err)
|
||||
}
|
||||
@@ -122,6 +143,70 @@ func ConfigSources(_ context.Context, args []string, out io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigArtifacts validates and reports the effective concrete artifact catalog
|
||||
// without loading session state or runtime collaborators.
|
||||
func ConfigArtifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configArtifactsWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configArtifactsWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config artifacts", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config artifacts: %w", err)
|
||||
}
|
||||
records, err := config.EffectiveArtifactCatalog(resolved.Pipeline, resolved.Party)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config artifacts: %w", err)
|
||||
}
|
||||
if _, err := fmt.Fprintln(out, "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range records {
|
||||
fields := []string{
|
||||
record.Key,
|
||||
record.SourceID,
|
||||
strings.Join(normalizedCatalogPaths(record.DefinitionSources), ","),
|
||||
strconv.FormatBool(record.Enabled),
|
||||
record.OutputPath,
|
||||
strings.Join(record.Dependencies, ","),
|
||||
record.FamilyKey,
|
||||
record.CharacterID,
|
||||
config.NormalizedConfigurationPath(record.FamilySource),
|
||||
config.NormalizedConfigurationPath(record.PartySource),
|
||||
}
|
||||
for index := range fields {
|
||||
fields[index] = escapeCatalogField(fields[index])
|
||||
}
|
||||
if _, err := fmt.Fprintln(out, strings.Join(fields, "\t")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizedCatalogPaths(paths []string) []string {
|
||||
result := make([]string, len(paths))
|
||||
for index, path := range paths {
|
||||
result[index] = config.NormalizedConfigurationPath(path)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func escapeCatalogField(value string) string {
|
||||
value = strings.ReplaceAll(value, "\\", "\\\\")
|
||||
value = strings.ReplaceAll(value, "\t", "\\t")
|
||||
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||
return strings.ReplaceAll(value, "\n", "\\n")
|
||||
}
|
||||
|
||||
func parseInspectionFlags(command string, args []string, out io.Writer) (inspectionFlags, error) {
|
||||
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
@@ -131,8 +216,10 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
||||
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||
flags.profile.name = "profile"
|
||||
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
||||
flags.secretsDir.name = "secrets-dir"
|
||||
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]\n\n", command)
|
||||
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <path>]\n\n", command)
|
||||
fmt.Fprintln(out, "Resolves configuration without a session, workspace, manifest, adapters, or external services.")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Flags:")
|
||||
@@ -150,8 +237,8 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func resolveInspectionConfig(flags inspectionFlags) (*inspectionConfig, error) {
|
||||
pipelinePath, pipeline, err := loadPipelineConfig(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()})
|
||||
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
|
||||
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -236,9 +323,9 @@ func inspectionProfile(pipeline *config.PipelineConfig) string {
|
||||
}
|
||||
|
||||
func writeConfigCommandUsage(out io.Writer) {
|
||||
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|diff>")
|
||||
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|artifacts|diff>")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, or config diff to compare two profiles.")
|
||||
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, config artifacts to list concrete artifacts, or config diff to compare two profiles.")
|
||||
}
|
||||
|
||||
func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) error {
|
||||
@@ -274,7 +361,7 @@ func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) erro
|
||||
if _, err := fmt.Fprintf(out, "campaign: id=%s source=%s\n", config.CampaignID(resolved.Campaign), config.NormalizedConfigurationPath(resolved.CampaignPath)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "party: mode=%s source=%s\n", resolved.Party.Mode, config.NormalizedConfigurationPath(resolved.Party.Source.Path)); err != nil {
|
||||
if _, err := fmt.Fprintf(out, "party: source=%s\n", config.NormalizedConfigurationPath(resolved.Party.Source.Path)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
@@ -21,13 +22,14 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
|
||||
want string
|
||||
code int
|
||||
}{
|
||||
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|diff>"},
|
||||
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|artifacts|diff>"},
|
||||
{name: "validate help", args: []string{"config", "validate", "--help"}, want: "Usage: narratio config validate"},
|
||||
{name: "show help", args: []string{"config", "show", "--help"}, want: "Usage: narratio config show"},
|
||||
{name: "sources help", args: []string{"config", "sources", "--help"}, want: "Usage: narratio config sources"},
|
||||
{name: "artifacts help", args: []string{"config", "artifacts", "--help"}, want: "Usage: narratio config artifacts"},
|
||||
{name: "diff help", args: []string{"config", "diff", "--help"}, want: "Usage: narratio config diff"},
|
||||
{name: "unknown", args: []string{"config", "unknown"}, want: `config: unknown subcommand "unknown"`, code: 1},
|
||||
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|diff", code: 1},
|
||||
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|artifacts|diff", code: 1},
|
||||
{name: "session flag", args: []string{"config", "validate", "--session", "session.yml"}, want: "flag provided but not defined", code: 1},
|
||||
{name: "stage flag", args: []string{"config", "show", "--from", "prepare"}, want: "flag provided but not defined", code: 1},
|
||||
{name: "force flag", args: []string{"config", "show", "--force"}, want: "flag provided but not defined", code: 1},
|
||||
@@ -47,6 +49,38 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteConfigUsesInvocationPipelineLoader(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
|
||||
for _, subcommand := range []string{"validate", "show", "sources", "artifacts"} {
|
||||
t.Run(subcommand, func(t *testing.T) {
|
||||
calls := 0
|
||||
deps := commandDependencies{
|
||||
loadPipelineConfig: func(path string, opts config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
calls++
|
||||
return config.LoadPipelineWithOptions(path, opts)
|
||||
},
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := executeWithDependencies(
|
||||
[]string{"config", subcommand, "--config", pipelinePath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
deps,
|
||||
executePlan,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("pipeline loader calls = %d, want 1", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
workspaceRoot := filepath.Join(dir, "workspace-does-not-exist")
|
||||
@@ -59,8 +93,6 @@ func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
|
||||
defer server.Close()
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: `+workspaceRoot+`
|
||||
secrets:
|
||||
env_dir: `+secretsDir+`
|
||||
whisperx:
|
||||
transcribe_url: `+server.URL+`
|
||||
`)
|
||||
@@ -116,6 +148,8 @@ func TestConfigDiffReportsSemanticProfileChanges(t *testing.T) {
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
writeInspectionProfiles(t, pipelinePath, `workspace:
|
||||
cleanup_after_publish: true
|
||||
publish:
|
||||
upload_run: true
|
||||
whisperx:
|
||||
language: en
|
||||
audita:
|
||||
@@ -173,6 +207,56 @@ scriptorium:
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: /tmp/narratio-work
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
audita:
|
||||
modules: null
|
||||
scriptorium:
|
||||
artifacts: null
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "profiles", "empty.yml"), `audita:
|
||||
modules: []
|
||||
scriptorium:
|
||||
artifacts: {}
|
||||
`)
|
||||
|
||||
var base, empty bytes.Buffer
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "empty"}, &empty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"modules: null\n", "artifacts: null\n"} {
|
||||
if !strings.Contains(base.String(), want) {
|
||||
t.Fatalf("base show output missing %q:\n%s", want, base.String())
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"modules: []\n", "artifacts: {}\n"} {
|
||||
if !strings.Contains(empty.String(), want) {
|
||||
t.Fatalf("empty-profile show output missing %q:\n%s", want, empty.String())
|
||||
}
|
||||
}
|
||||
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"changed\taudita.modules\tnull\t[]\n",
|
||||
"changed\tscriptorium.artifacts\tnull\t{}\n",
|
||||
} {
|
||||
if !strings.Contains(diff.String(), want) {
|
||||
t.Fatalf("diff output missing %q:\n%s", want, diff.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
writeInspectionProfiles(t, pipelinePath, `scriptorium:
|
||||
@@ -242,6 +326,137 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiffBootstrapsSecretsOnceForBothProfiles(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
writeInspectionProfiles(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
|
||||
secretsDir := t.TempDir()
|
||||
const secretName = "NARRATIO_DIFF_BOOTSTRAP_REPORT"
|
||||
restoreEnvAfterTest(t, secretName)
|
||||
if err := os.Unsetenv(secretName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretName), "diff-secret-must-not-appear\n")
|
||||
bootstrapCalls := 0
|
||||
var diagnostics bytes.Buffer
|
||||
deps := commandDependencies{loadSecretsDir: func(got string) (*configkit.SecretsReport, error) {
|
||||
bootstrapCalls++
|
||||
if got != secretsDir {
|
||||
t.Fatalf("secrets directory = %q, want %q", got, secretsDir)
|
||||
}
|
||||
return configkit.LoadSecretsDir(got)
|
||||
}, diagnostics: &diagnostics}
|
||||
|
||||
if err := configDiffWithDependencies(context.Background(), []string{
|
||||
"production", "testing", "--config", pipelinePath, "--secrets-dir", secretsDir,
|
||||
}, io.Discard, deps); err != nil {
|
||||
t.Fatalf("config diff: %v", err)
|
||||
}
|
||||
if bootstrapCalls != 1 {
|
||||
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
|
||||
}
|
||||
if got := diagnostics.String(); strings.Count(got, `name="`+secretName+`"`) != 1 {
|
||||
t.Fatalf("bootstrap report count = %d, want 1; diagnostics = %q", strings.Count(got, `name="`+secretName+`"`), got)
|
||||
}
|
||||
if strings.Contains(diagnostics.String(), "diff-secret-must-not-appear") {
|
||||
t.Fatalf("bootstrap diagnostics exposed secret value: %q", diagnostics.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T) {
|
||||
const firstCanary = "inspection-credential-canary-one"
|
||||
const secondCanary = "inspection-credential-canary-two"
|
||||
for name, value := range map[string]string{
|
||||
"NARRATIO_INSPECT_ACCESS": firstCanary,
|
||||
"NARRATIO_INSPECT_SECRET": firstCanary,
|
||||
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
|
||||
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
|
||||
"NARRATIO_INSPECT_NOTARIUS_ONE": firstCanary,
|
||||
"NARRATIO_INSPECT_NOTARIUS_TWO": secondCanary,
|
||||
} {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelineData, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipelineYAML := strings.Replace(string(pipelineData), " bucket: test-bucket\n", ` bucket: test-bucket
|
||||
access_key_id: NARRATIO_INSPECT_ACCESS
|
||||
secret_access_key: NARRATIO_INSPECT_SECRET
|
||||
`, 1)
|
||||
pipelineYAML += `notarius:
|
||||
enabled: false
|
||||
llm_api_key: NARRATIO_INSPECT_NOTARIUS_ONE
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeInspectionProfiles(t, pipelinePath,
|
||||
"audita:\n model: production\n llm_api_key: NARRATIO_INSPECT_LLM_ONE\n",
|
||||
"audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\nnotarius:\n llm_api_key: NARRATIO_INSPECT_NOTARIUS_TWO\n",
|
||||
)
|
||||
mustWriteTestFile(t, filepath.Join(filepath.Dir(pipelinePath), "profiles", "without-notarius-key.yml"), "notarius:\n llm_api_key: null\n")
|
||||
|
||||
var validate bytes.Buffer
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &validate); err != nil {
|
||||
t.Fatalf("config validate: %v", err)
|
||||
}
|
||||
|
||||
var show bytes.Buffer
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &show); err != nil {
|
||||
t.Fatalf("config show: %v", err)
|
||||
}
|
||||
var sources bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &sources); err != nil {
|
||||
t.Fatalf("config sources: %v", err)
|
||||
}
|
||||
var catalog bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &catalog); err != nil {
|
||||
t.Fatalf("config artifacts: %v", err)
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil {
|
||||
t.Fatalf("config diff: %v", err)
|
||||
}
|
||||
var removed bytes.Buffer
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "without-notarius-key"}, &removed); err != nil {
|
||||
t.Fatalf("config show with removed Notarius credential: %v", err)
|
||||
}
|
||||
for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} {
|
||||
if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) {
|
||||
t.Fatalf("%s output exposed credential value: %q", label, output)
|
||||
}
|
||||
}
|
||||
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE", "NARRATIO_INSPECT_NOTARIUS_ONE"} {
|
||||
if !strings.Contains(show.String(), reference) {
|
||||
t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String())
|
||||
}
|
||||
}
|
||||
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") ||
|
||||
!strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_TWO") {
|
||||
t.Fatalf("diff omitted safe credential references: %q", diff.String())
|
||||
}
|
||||
if !strings.Contains(sources.String(), "notarius.llm_api_key") {
|
||||
t.Fatalf("sources omitted Notarius credential ownership: %q", sources.String())
|
||||
}
|
||||
if strings.Contains(removed.String(), "llm_api_key") || strings.Contains(removed.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") {
|
||||
t.Fatalf("profile null did not remove Notarius credential: %q", removed.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateRejectsUnavailableNotariusCredential(t *testing.T) {
|
||||
const envName = "NARRATIO_INSPECT_NOTARIUS_MISSING"
|
||||
restoreEnvAfterTest(t, envName)
|
||||
path := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||
mustWriteTestFile(t, path, "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n")
|
||||
err := ConfigValidate(context.Background(), []string{"--config", path}, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "secret.unset") {
|
||||
t.Fatalf("ConfigValidate() error = %v, want unavailable credential failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
|
||||
examplesDir := filepath.Join("..", "..", "examples")
|
||||
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
||||
@@ -251,13 +466,13 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
||||
t.Fatalf("example workspace stat = %v, want absent", err)
|
||||
}
|
||||
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} {
|
||||
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil {
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources, ConfigArtifacts} {
|
||||
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, io.Discard); err != nil {
|
||||
t.Fatalf("inspection command %T error = %v", command, err)
|
||||
}
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||
if err := ConfigDiff(context.Background(), []string{"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||
t.Fatalf("ConfigDiff() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(diff.String(), "audita.model") {
|
||||
@@ -270,17 +485,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
||||
|
||||
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Dir(pipelinePath)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
|
||||
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
|
||||
}
|
||||
|
||||
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
@@ -292,7 +499,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
|
||||
t.Fatalf("default ConfigValidate() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(defaultOut.String(), "profile=production") {
|
||||
if !strings.Contains(defaultOut.String(), "profile=none") {
|
||||
t.Fatalf("default output = %q", defaultOut.String())
|
||||
}
|
||||
|
||||
@@ -318,7 +525,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
func TestConfigCommandsExpandFamiliesAndRequireCampaign(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow} {
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigArtifacts} {
|
||||
var out bytes.Buffer
|
||||
err := command(context.Background(), []string{"--config", pipelinePath}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "artifact_families requires a campaign") {
|
||||
@@ -364,8 +571,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
|
||||
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [conf.yml]
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: conf.yml
|
||||
workspace:
|
||||
root: /one
|
||||
whisperx:
|
||||
@@ -373,7 +581,7 @@ whisperx:
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
|
||||
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
t.Fatalf("import diagnostic = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -398,14 +606,14 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"root: ",
|
||||
"imports: none",
|
||||
"profile: name=testing selection=cli overlay=",
|
||||
"profile: name=testing selection=explicit overlay=",
|
||||
"campaign: id=sample-campaign source=",
|
||||
"party: mode=legacy source=",
|
||||
"party: source=",
|
||||
"digest: ",
|
||||
"path\trole\tsource\n",
|
||||
"workspace.root\troot\t",
|
||||
"whisperx.language\tprofile\t",
|
||||
"campaign.inputs.players_file\tlegacy_player\t",
|
||||
"derived.players\tparty\t",
|
||||
"trim.enabled\tdefault\tdefault",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
@@ -423,7 +631,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
||||
output := out.String()
|
||||
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
|
||||
for _, want := range []string{
|
||||
"party: mode=canonical source=" + partyPath,
|
||||
"party: source=" + partyPath,
|
||||
"derived.players\tparty\t" + partyPath,
|
||||
"scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + pipelinePath,
|
||||
"scriptorium.artifacts.character_note_arannis.enabled\tparty\t" + partyPath,
|
||||
@@ -441,15 +649,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
||||
|
||||
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
|
||||
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifact_families:
|
||||
character_note:
|
||||
@@ -458,7 +658,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
|
||||
@@ -467,17 +667,74 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesAttributesInferredArtifactDependencies(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
inputPath := filepath.Join(dir, "input-only.yml")
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `configkit:
|
||||
imports:
|
||||
- path: input-only.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
scriptorium:
|
||||
artifacts:
|
||||
first:
|
||||
enabled: false
|
||||
output_path: artifacts/first.md
|
||||
second:
|
||||
enabled: false
|
||||
output_path: artifacts/second.md
|
||||
mixed:
|
||||
enabled: false
|
||||
output_path: artifacts/mixed.md
|
||||
depends_on: [first]
|
||||
`)
|
||||
mustWriteTestFile(t, inputPath, `scriptorium:
|
||||
artifacts:
|
||||
input_only:
|
||||
enabled: false
|
||||
output_path: artifacts/input-only.md
|
||||
inputs:
|
||||
prior:
|
||||
source: narratio.artifact.first
|
||||
`)
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifacts:
|
||||
mixed:
|
||||
inputs:
|
||||
prior:
|
||||
source: narratio.artifact.second
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"scriptorium.artifacts.input_only.depends_on\timport\t" + inputPath,
|
||||
"scriptorium.artifacts.mixed.depends_on\troot\t" + pipelinePath,
|
||||
"scriptorium.artifacts.mixed.depends_on\tprofile\t" + profilePath,
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Fatalf("sources output missing dependency contributor %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
if strings.Contains(out.String(), "scriptorium.artifacts.input_only.depends_on\tdefault\tdefault") {
|
||||
t.Fatalf("inferred dependency retained default attribution:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
importPath := filepath.Join(dir, "base.yml")
|
||||
profilePath := filepath.Join(dir, "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [base.yml]
|
||||
default_profile: testing
|
||||
profiles:
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: base.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
@@ -487,13 +744,13 @@ whisperx:
|
||||
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"import: " + importPath,
|
||||
"profile: name=testing selection=default overlay=" + profilePath,
|
||||
"profile: name=testing selection=explicit overlay=" + profilePath,
|
||||
"workspace.root\troot\t" + rootPath,
|
||||
"storage.backend\timport\t" + importPath,
|
||||
"whisperx.language\tprofile\t" + profilePath,
|
||||
|
||||
@@ -16,6 +16,7 @@ type configDiffFlags struct {
|
||||
pipelinePath singletonStringFlag
|
||||
campaignPath singletonStringFlag
|
||||
campaignFilePath singletonStringFlag
|
||||
secretsDir singletonStringFlag
|
||||
}
|
||||
|
||||
type configDiffRequest struct {
|
||||
@@ -40,7 +41,11 @@ type configDiffRecord struct {
|
||||
// read-only configuration resolution boundary used by config validate, show,
|
||||
// and sources. Differences describe normalized effective values rather than
|
||||
// the layout of root, import, or overlay files.
|
||||
func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigDiff(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configDiffWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configDiffWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
request, err := parseConfigDiffRequest(args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -48,7 +53,7 @@ func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveConfigDiff(request)
|
||||
resolved, err := resolveConfigDiff(request, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config diff: %w", err)
|
||||
}
|
||||
@@ -102,11 +107,13 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
||||
request.flags.pipelinePath.name = "config"
|
||||
request.flags.campaignPath.name = "campaign"
|
||||
request.flags.campaignFilePath.name = "campaign-file"
|
||||
request.flags.secretsDir.name = "secrets-dir"
|
||||
fs.Var(&request.flags.pipelinePath, "config", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.Var(&request.flags.campaignPath, "campaign", "campaign ID")
|
||||
fs.Var(&request.flags.campaignFilePath, "campaign-file", "path to campaign.yml")
|
||||
fs.Var(&request.flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]")
|
||||
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <path>]")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Compares fully resolved profiles without a session, workspace, manifest, adapters, or external services.")
|
||||
fmt.Fprintln(out)
|
||||
@@ -120,6 +127,9 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
||||
if len(profiles) != 2 || strings.TrimSpace(profiles[0]) == "" || strings.TrimSpace(profiles[1]) == "" {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty profile names are required")
|
||||
}
|
||||
if profiles[0] == profiles[1] {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: profile names must differ")
|
||||
}
|
||||
if request.flags.campaignPath.set && request.flags.campaignFilePath.set {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: --campaign and --campaign-file are mutually exclusive")
|
||||
}
|
||||
@@ -156,21 +166,22 @@ func splitConfigDiffArguments(args []string) (profiles, flagArgs []string) {
|
||||
|
||||
func configDiffStringFlag(name string) bool {
|
||||
switch name {
|
||||
case "config", "campaign", "campaign-file", "profile":
|
||||
case "config", "campaign", "campaign-file", "profile", "secrets-dir":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func resolveConfigDiff(request configDiffRequest) (*configDiffResolved, error) {
|
||||
pipelinePath, err := resolvePipelineConfigPath(request.flags.pipelinePath.value)
|
||||
func resolveConfigDiff(request configDiffRequest, deps commandDependencies) (*configDiffResolved, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
pipelinePath, leftPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.leftProfile}, request.flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
leftPipeline, rightPipeline, err := config.LoadPipelineProfilePair(pipelinePath, request.leftProfile, request.rightProfile)
|
||||
_, rightPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.rightProfile}, request.flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
needsCampaign := pipelineHasArtifactFamilies(leftPipeline) || pipelineHasArtifactFamilies(rightPipeline)
|
||||
hasCampaignSelection := request.flags.campaignPath.set || request.flags.campaignFilePath.set
|
||||
|
||||
@@ -16,9 +16,6 @@ import (
|
||||
|
||||
type pipelineCampaignConfig = config.LoadedPipelineCampaign
|
||||
|
||||
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
||||
var loadPipelineConfigFn = config.LoadPipelineWithOptions
|
||||
|
||||
type commandConfig struct {
|
||||
Config *config.Config
|
||||
cleanup func() error
|
||||
@@ -37,7 +34,8 @@ func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
||||
return &commandConfig{Config: cfg}
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (loaded *commandConfig, err error) {
|
||||
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (loaded *commandConfig, err error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
var cleanup func() error
|
||||
defer func() {
|
||||
if err == nil || cleanup == nil {
|
||||
@@ -48,7 +46,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
}
|
||||
}()
|
||||
|
||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile})
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, secretsDir, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -88,7 +86,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, partialCfg, nil, deps)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
@@ -97,7 +95,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := downloadObjectToTempFn(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
sessionTempPath, err := deps.downloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
@@ -137,8 +135,9 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions) (*pipelineCampaignConfig, error) {
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfig(pipelineFlag, pipelineOpts)
|
||||
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -162,12 +161,20 @@ func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag str
|
||||
return &loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineConfig(pipelineFlag string, opts config.PipelineLoadOptions) (string, *config.PipelineConfig, error) {
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
resolvedPipelinePath, err := deps.resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pipelineCfg, err := loadPipelineConfigFn(resolvedPipelinePath, opts)
|
||||
if !deps.secretsBootstrap.selectionSet {
|
||||
deps.secretsBootstrap.selection = secretsDir
|
||||
deps.secretsBootstrap.selectionSet = true
|
||||
}
|
||||
if err := deps.secretsBootstrap.load(resolvedPipelinePath, deps.loadSecretsDir, deps.diagnostics); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user