Compare commits
162 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 792cbd0677 | |||
| 5dca34e0e0 | |||
| 54d335c820 | |||
| 96b4e73f53 | |||
| 6e6054f744 | |||
| dc3288d15a | |||
| d872edca94 | |||
| 2ae63dbc2b | |||
| 26aa0cce29 | |||
| 7b0bf99ed1 | |||
| 89ada1b269 | |||
| 3438fe82c9 | |||
| a51a513804 | |||
| 995354f1a5 | |||
| 931c9591cd | |||
| 46096fc4ba | |||
| d3ecd907ee | |||
| 442469b9f5 | |||
| c45e042462 | |||
| 207da1b6ef | |||
| 9a1609435e | |||
| 13b83d7ee2 | |||
| a0312ed034 | |||
| 8ff74e1251 | |||
| 1b480d4ed5 | |||
| e13d87a077 | |||
| 5202f16972 | |||
| 5086596f94 | |||
| f7fc64db10 | |||
| 6b0778b420 | |||
| 5b92b3e790 | |||
| cc43906ae9 | |||
| dcaa2ad9c6 | |||
| 9d68cfcf2f | |||
| 2d346733d6 | |||
| cd75bf6d5a | |||
| e08de98ae6 | |||
| df213bc2ba | |||
| 869e68e834 | |||
| ed4623bc41 | |||
| 3ce198197d | |||
| 0ac8072a9a | |||
| 3ec54b8c72 | |||
| 47b3e5b55b | |||
| 99d1a0b926 | |||
| 884a62e8a6 | |||
| f7a40afc7a | |||
| a4f37ff427 | |||
| 2b97ef3722 | |||
| 63a86c43f4 | |||
| 92432232ef | |||
| d31ba5a410 | |||
| f0225e937c | |||
| 9e40c729e1 | |||
| 0e6ba414b3 | |||
| 10a2e89965 | |||
| f8e3e651e6 | |||
| 1558c9ccae | |||
| b68435128b | |||
| 5fb454a714 | |||
| 79cf9f1139 | |||
| 9bb203476d | |||
| 0726786016 | |||
| 204caf9d24 | |||
| 85663f5bc9 | |||
| f51b9bb046 | |||
| dd905c4daa | |||
| c3307ab5fb | |||
| 1484f51f48 | |||
| 51d21676c1 | |||
| c2806a1615 | |||
| 078f661e3a | |||
| adc60aa588 | |||
| 8d9eb5cc40 | |||
| 563acce012 | |||
| 49c359feb8 | |||
| 38202d4472 | |||
| 6d07dafc9f | |||
| 6c66e18c4b | |||
| 4048996652 | |||
| 470450e509 | |||
| 7e78c22fec | |||
| 610eaefb27 | |||
| 4bc33773fa | |||
| b74e31e1cc | |||
| 2f1e4407db | |||
| 4ccb4a73ca | |||
| 145baac393 | |||
| be59bff690 | |||
| 14b4fee214 | |||
| 7f84fc5d1e | |||
| a8b47f6185 | |||
| 5b62a8d60d | |||
| 0d4d71b9dc | |||
| 979d8049b1 | |||
| abd405015b | |||
| fddf78dc80 | |||
| 3136c26d4e | |||
| 4b84b4223d | |||
| 7a0e2cb11b | |||
| 74ab9218ff | |||
| da99857fd0 | |||
| a520cfdcc8 | |||
| 7febccf550 | |||
| 74977b4064 | |||
| 172e2d6a4f | |||
| 5c520dc0d4 | |||
| 6d18a3126e | |||
| 1e222a4db3 | |||
| 6a15052d77 | |||
| 2e69c14898 | |||
| dbda2a0161 | |||
| 2ee64f2eb1 | |||
| 8afe9427ca | |||
| e0f19833a1 | |||
| ca1b694aa9 | |||
| 9331373245 | |||
| ec072dc163 | |||
| f7ce1dcd56 | |||
| df275874e3 | |||
| 1da34ab5cd | |||
| c0fbc14f2d | |||
| 2d20900cbf | |||
| 7997f81f47 | |||
| 6d748d65d3 | |||
| 49324d3f35 | |||
| c98b4ef4ef | |||
| c8bae3a4a3 | |||
| ccbd7ee820 | |||
| e9c4873030 | |||
| 57719b7d81 | |||
| 62b1f9b0ae | |||
| 6c7a45d8d0 | |||
| 82d75e7beb | |||
| 61095e69ce | |||
| 2cfca710ec | |||
| 9679117e94 | |||
| d67a0389e0 | |||
| 9f84446509 | |||
| ec9b0e919d | |||
| 9bb7c1f4e3 | |||
| 3c092fbe34 | |||
| 3a7e2e5175 | |||
| 45b27ad562 | |||
| 1e418d6929 | |||
| 39b2665d8b | |||
| ba858cc997 | |||
| 0d57422b61 | |||
| b7d989f61f | |||
| 6813340f51 | |||
| eb788f17b8 | |||
| ebd3f415d3 | |||
| f15d8d8ce0 | |||
| 6c7bfd5e31 | |||
| 7c782a2603 | |||
| f1eef5b5e9 | |||
| f14583321d | |||
| 04283e99d4 | |||
| 7284228bec | |||
| da74d7ad18 | |||
| af5fbc1c4f | |||
| 5e4d2016e8 |
@@ -3,12 +3,12 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
validate-release:
|
validate-release:
|
||||||
image: golang:1.25.5
|
image: golang:1.26.7
|
||||||
commands:
|
commands:
|
||||||
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
|
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
|
||||||
|
|
||||||
build-release-assets:
|
build-release-assets:
|
||||||
image: golang:1.25.5
|
image: golang:1.26.7
|
||||||
depends_on:
|
depends_on:
|
||||||
- validate-release
|
- validate-release
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
@@ -3,6 +3,6 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
shuffled-race-tests:
|
shuffled-race-tests:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
commands:
|
commands:
|
||||||
- go test -race -shuffle=on -count=3 ./...
|
- go test -race -shuffle=on -count=3 ./...
|
||||||
|
|||||||
@@ -3,44 +3,44 @@ when:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
tests:
|
tests:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
commands:
|
commands:
|
||||||
- go test ./...
|
- go test ./...
|
||||||
|
|
||||||
race-tests:
|
race-tests:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
depends_on: tests
|
depends_on: tests
|
||||||
commands:
|
commands:
|
||||||
- go test -race ./...
|
- go test -race ./...
|
||||||
|
|
||||||
static-analysis:
|
static-analysis:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
depends_on: tests
|
depends_on: tests
|
||||||
commands:
|
commands:
|
||||||
- go vet ./...
|
- go vet ./...
|
||||||
|
|
||||||
build:
|
build:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
depends_on: tests
|
depends_on: tests
|
||||||
commands:
|
commands:
|
||||||
- go build ./...
|
- go build ./...
|
||||||
|
|
||||||
documentation-and-examples:
|
documentation-and-examples:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
depends_on: tests
|
depends_on: tests
|
||||||
commands:
|
commands:
|
||||||
- go test ./internal/doccheck
|
- go test ./internal/doccheck
|
||||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
|
|
||||||
cross-build:
|
cross-build:
|
||||||
image: golang:1.25
|
image: golang:1.26
|
||||||
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set -eu
|
set -eu
|
||||||
output_dir="$(mktemp -d)"
|
output_dir="$(mktemp -d)"
|
||||||
trap 'rm -rf "$output_dir"' EXIT
|
trap 'rm -rf "$output_dir"' EXIT
|
||||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
for target in linux/amd64 linux/arm64 darwin/arm64; do
|
||||||
goos="${target%/*}"
|
goos="${target%/*}"
|
||||||
goarch="${target#*/}"
|
goarch="${target#*/}"
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||||
|
|||||||
104
docs/cli.md
104
docs/cli.md
@@ -20,7 +20,7 @@ Top-level commands:
|
|||||||
- `publish <session_id>`: force-run publish.
|
- `publish <session_id>`: force-run publish.
|
||||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||||
- `session <subcommand>`: session helper commands.
|
- `session <subcommand>`: session helper commands.
|
||||||
- `config <subcommand>`: validate, display, source-trace, or compare resolved pipeline configuration.
|
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
|
||||||
|
|
||||||
Session subcommands:
|
Session subcommands:
|
||||||
|
|
||||||
@@ -45,6 +45,7 @@ Most session-aware commands accept:
|
|||||||
- `--session-id <session_id>`
|
- `--session-id <session_id>`
|
||||||
- `--previous-session-id <session_id>`
|
- `--previous-session-id <session_id>`
|
||||||
- `--profile <name>`
|
- `--profile <name>`
|
||||||
|
- `--secrets-dir <directory>`
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
|
|
||||||
@@ -53,10 +54,13 @@ Rules:
|
|||||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||||
- `--previous-session-id` is a strict expectation: the selected session file
|
- `--previous-session-id` is a strict expectation: the selected session file
|
||||||
must contain the same `previous_session_id`.
|
must contain the same `previous_session_id`.
|
||||||
- `--profile` selects a declared pipeline profile. It may be supplied once;
|
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
|
||||||
an explicit empty or unknown value fails configuration resolution. When it is
|
may be supplied once; omitted, empty, and the exact name `default` select the
|
||||||
omitted, a declared `default_profile` is used. The same selection applies to
|
base configuration, while an unknown named profile fails. The same selection
|
||||||
all common-flag commands, including `regenerate-artifacts`.
|
applies to all common-flag commands, including `regenerate-artifacts`.
|
||||||
|
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
|
||||||
|
relative value uses caller CWD. Without the flag, a present `secrets/`
|
||||||
|
directory beside the resolved root pipeline is used; an absent one is skipped.
|
||||||
- `clean --all` cannot be combined with campaign/session selectors.
|
- `clean --all` cannot be combined with campaign/session selectors.
|
||||||
- notification delivery is currently limited to the configured `noop` mode; see
|
- notification delivery is currently limited to the configured `noop` mode; see
|
||||||
the [configuration reference](./config.md#notifications).
|
the [configuration reference](./config.md#notifications).
|
||||||
@@ -78,19 +82,24 @@ Commands with additional positionals keep their command-specific order:
|
|||||||
|
|
||||||
## Command Reference
|
## Command Reference
|
||||||
|
|
||||||
### `config validate`, `config show`, `config sources`, and `config diff`
|
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]
|
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
|
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
|
||||||
```
|
```
|
||||||
|
|
||||||
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
||||||
a campaign is selected—the campaign-owned party. They neither discover or load
|
a campaign is selected—the campaign-owned party. They neither discover or load
|
||||||
a session nor create a workspace, manifest, run, lock, adapter, remote
|
a session nor create a workspace, manifest, run, lock, adapter, or remote
|
||||||
connection, or credential environment.
|
connection. Like every pipeline-loading command, they bootstrap the selected or
|
||||||
|
conventional secrets directory before resolving configuration; `config show`
|
||||||
|
never includes bootstrap reports or secret values in its YAML output.
|
||||||
|
Typed credential references, including `notarius.llm_api_key`, must therefore
|
||||||
|
resolve even though these inspection commands never construct an adapter.
|
||||||
|
|
||||||
Campaign selection is optional for a pipeline without party-driven artifact
|
Campaign selection is optional for a pipeline without party-driven artifact
|
||||||
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
||||||
@@ -101,20 +110,31 @@ Session, range, force, and artifact-execution flags are not accepted.
|
|||||||
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
||||||
effective-digest summary after successful complete validation. `config show`
|
effective-digest summary after successful complete validation. `config show`
|
||||||
writes one deterministic, secret-free YAML document containing defaulted and
|
writes one deterministic, secret-free YAML document containing defaulted and
|
||||||
expanded concrete configuration. It omits composition declarations, artifact
|
expanded concrete configuration. It omits ConfigKit directives, artifact
|
||||||
family declarations, and runtime provenance.
|
family declarations, and runtime provenance.
|
||||||
|
|
||||||
`config sources` reports the same fully validated resolution without printing
|
`config sources` reports the same fully validated resolution without printing
|
||||||
effective values. Its header identifies the root, ordered imports, selected
|
effective values. Its header identifies the root, ordered imports, selected
|
||||||
profile and overlay, selected campaign, party mode/source, and digest. The
|
profile and overlay, selected campaign, party source, and digest. The
|
||||||
remaining tab-separated records are sorted as `path`, `role`, and `source`.
|
remaining tab-separated records are sorted as `path`, `role`, and `source`.
|
||||||
Roles distinguish root, import, profile, centralized default, campaign, party,
|
Roles distinguish root, import, profile, centralized default, campaign, party,
|
||||||
legacy-player, and generated family ownership. A generated party member has
|
and generated family ownership. A generated party member has
|
||||||
one family record and one party record at the same logical path. The output
|
one family record and one party record at the same logical path. The output
|
||||||
never reads or prints secret values.
|
never reads or prints secret values.
|
||||||
|
|
||||||
`config diff` resolves both supplied profile names from one parsed root source
|
`config artifacts` prints the concrete catalog used by analysis, ordered by
|
||||||
set and compares their fully resolved, secret-free effective mappings. It does
|
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
||||||
|
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||||
|
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||||
|
lists, and control characters in fields are escaped. Family-member rows identify
|
||||||
|
their family, character, defining family file, and party file. An empty catalog
|
||||||
|
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
||||||
|
party-driven families retain the campaign requirement. See
|
||||||
|
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||||
|
configuration layout.
|
||||||
|
|
||||||
|
`config diff` resolves both supplied profile names independently and compares
|
||||||
|
their fully resolved, secret-free effective mappings. It does
|
||||||
not accept `--profile`; the two positional names must be distinct, declared
|
not accept `--profile`; the two positional names must be distinct, declared
|
||||||
profiles. When party-driven families are present, both profiles must resolve to
|
profiles. When party-driven families are present, both profiles must resolve to
|
||||||
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
||||||
@@ -153,17 +173,22 @@ Behavior:
|
|||||||
|
|
||||||
- evaluates one inclusive contiguous range of the canonical stage order;
|
- evaluates one inclusive contiguous range of the canonical stage order;
|
||||||
- defaults an omitted `--from` to `prepare` and an omitted `--through` to
|
- defaults an omitted `--from` to `prepare` and an omitted `--through` to
|
||||||
`notify`, so omitting both retains full-pipeline behavior;
|
`cleanup`, so omitting both retains full-pipeline behavior;
|
||||||
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
||||||
- runs `render` before `extract`; an omitted or disabled Notarius
|
- runs `render` before `extract`; an omitted or disabled Notarius
|
||||||
configuration records an explicit `notarius_disabled` self-skip;
|
configuration records an explicit `notarius_disabled` self-skip;
|
||||||
|
- resolves every selected typed credential during the initial configuration
|
||||||
|
load, including a Notarius credential when extract is disabled, outside the
|
||||||
|
selected range, or later skipped as resumable;
|
||||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||||
resume check finds its durable result obsolete;
|
resume check finds its durable result obsolete;
|
||||||
- applies `--force` only to stages in the selected range;
|
- applies `--force` only to stages in the selected range;
|
||||||
- rejects repeated `--from`, `--through`, or `--force` options, including
|
- rejects repeated `--from`, `--through`, or `--force` options, including
|
||||||
`--name=value` spellings;
|
`--name=value` spellings;
|
||||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- writes session and run manifests.
|
- writes the session manifest as the sole durable progress ledger. Each
|
||||||
|
invocation also creates its run-local directory and returns an immediate
|
||||||
|
command summary; neither creates another durable progress record.
|
||||||
- reports the resolved profile (or `none`) and effective configuration digest.
|
- reports the resolved profile (or `none`) and effective configuration digest.
|
||||||
|
|
||||||
When `--artifacts` is present, the selected range must contain `analyze` or
|
When `--artifacts` is present, the selected range must contain `analyze` or
|
||||||
@@ -187,6 +212,10 @@ artifacts and any prerequisites required by those targets; without
|
|||||||
never run. Common session/configuration options and repeatable artifact values
|
never run. Common session/configuration options and repeatable artifact values
|
||||||
pass through unchanged.
|
pass through unchanged.
|
||||||
|
|
||||||
|
The forced extraction rebuilds the complete configured Notarius bundle,
|
||||||
|
disables Notarius checkpoint loading, and requests chunk-plan refresh.
|
||||||
|
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
|
||||||
|
|
||||||
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
||||||
supply those options. The shared `run` parser reports them as duplicate
|
supply those options. The shared `run` parser reports them as duplicate
|
||||||
singleton flags. The alias has no private execution options or behavior, and
|
singleton flags. The alias has no private execution options or behavior, and
|
||||||
@@ -211,6 +240,7 @@ Valid stage names:
|
|||||||
- `analyze`
|
- `analyze`
|
||||||
- `publish`
|
- `publish`
|
||||||
- `notify`
|
- `notify`
|
||||||
|
- `cleanup`
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
|
|
||||||
@@ -228,6 +258,10 @@ Equivalent to:
|
|||||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This selects only `analyze`: it does not invoke Notarius, replace extraction,
|
||||||
|
or remove existing extract outputs. `--artifacts` narrows the forced analysis
|
||||||
|
targets and their required analysis prerequisites.
|
||||||
|
|
||||||
### `publish`
|
### `publish`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -240,6 +274,10 @@ Equivalent to:
|
|||||||
narratio run-stage publish <session_id> --force [...common config flags]
|
narratio run-stage publish <session_id> --force [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This command selects only `publish`; it does not invoke cleanup. Use a normal
|
||||||
|
full run to include configured cleanup, or `run-stage cleanup` to retry cleanup
|
||||||
|
from valid publication evidence.
|
||||||
|
|
||||||
### `clean`
|
### `clean`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -253,8 +291,8 @@ Behavior:
|
|||||||
- `--all` removes all local session work and spool state;
|
- `--all` removes all local session work and spool state;
|
||||||
- cache remains unless `--clear-cache` is provided.
|
- cache remains unless `--clear-cache` is provided.
|
||||||
|
|
||||||
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
|
See [Operations: Cleanup](./operations.md#cleanup) for manual and pipeline-stage
|
||||||
post-publish cleanup behavior.
|
deletion scope.
|
||||||
|
|
||||||
### `session plan`
|
### `session plan`
|
||||||
|
|
||||||
@@ -279,7 +317,8 @@ execution; the preview does not predict output identities.
|
|||||||
narratio session validate <session_id> [...common config flags]
|
narratio session validate <session_id> [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
Read-only preflight checks for config validity, required inputs, audio mode,
|
||||||
|
previous-session requirements, publish outputs, and independent remote locks.
|
||||||
|
|
||||||
### `session status`
|
### `session status`
|
||||||
|
|
||||||
@@ -288,7 +327,12 @@ narratio session status <session_id> [...common config flags]
|
|||||||
```
|
```
|
||||||
|
|
||||||
Prints local manifest state and, when storage is available, status for the
|
Prints local manifest state and, when storage is available, status for the
|
||||||
pointer-selected remote commit and its declared published outputs.
|
pointer-selected remote commit and its declared published outputs. Current
|
||||||
|
publish metadata reports `published`, `carried_locked`, `locked_unpublished`,
|
||||||
|
`optional_missing`, and `unselected`; carried output lines include the run
|
||||||
|
derived from their exact declared key, the key itself, and its declared
|
||||||
|
destination. A separately displayed `dest` is the destination requested by the
|
||||||
|
current configuration and may differ while an older declaration is locked.
|
||||||
|
|
||||||
### `session init`
|
### `session init`
|
||||||
|
|
||||||
@@ -307,6 +351,7 @@ Options:
|
|||||||
|
|
||||||
- `--config <pipeline.yml>`
|
- `--config <pipeline.yml>`
|
||||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||||
|
- `--secrets-dir <directory>`
|
||||||
- `--previous-session-id <id>`
|
- `--previous-session-id <id>`
|
||||||
- `--date <YYYY-MM-DD>`
|
- `--date <YYYY-MM-DD>`
|
||||||
- `--title <text>`
|
- `--title <text>`
|
||||||
@@ -346,7 +391,8 @@ narratio session artifacts <session_id> [--remote] [...common config flags]
|
|||||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||||
sources; reports planned, available, unavailable, and published state without
|
sources; reports planned, available, unavailable, and published state without
|
||||||
reading payload bodies; and includes publish rules, lock state, and optional
|
reading payload bodies; and includes publish rules, lock state, and optional
|
||||||
remote published-state availability.
|
remote published-state availability. Remote inspection uses exact commit
|
||||||
|
declarations and displays the five current publish outcomes when recorded.
|
||||||
|
|
||||||
### `session locks`
|
### `session locks`
|
||||||
|
|
||||||
@@ -358,12 +404,14 @@ narratio session locks remove <session_id> <source> [...common config flags]
|
|||||||
|
|
||||||
Behavior:
|
Behavior:
|
||||||
|
|
||||||
- list mode reports the effective merge of static and remote locks;
|
- list mode validates and reports independent remote locks sorted by source;
|
||||||
- add/remove mutate only remote locks;
|
- add creates one lock and reports an existing lock unless `--force` is used;
|
||||||
- static locks from pipeline config cannot be removed by CLI commands.
|
- `--force` performs one generation-matched replacement and reports a
|
||||||
|
concurrent change for operator retry; and
|
||||||
|
- remove is idempotent and reports when the source was not locked.
|
||||||
|
|
||||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||||
and precedence.
|
and publication behavior.
|
||||||
|
|
||||||
## `--artifacts` Selection Rules
|
## `--artifacts` Selection Rules
|
||||||
|
|
||||||
|
|||||||
292
docs/config.md
292
docs/config.md
@@ -44,13 +44,16 @@ the durable copied session input.
|
|||||||
|
|
||||||
### Read-only effective pipeline inspection
|
### Read-only effective pipeline inspection
|
||||||
|
|
||||||
`narratio config validate`, `narratio config show`, and `narratio config
|
`narratio config validate`, `narratio config show`, `narratio config sources`,
|
||||||
sources` use the same `--config`, `--campaign`, `--campaign-file`, and
|
and `narratio config artifacts` use the same `--config`, `--campaign`,
|
||||||
|
`--campaign-file`, and
|
||||||
`--profile` selection rules as pipeline commands, but do not select, discover,
|
`--profile` selection rules as pipeline commands, but do not select, discover,
|
||||||
or load a session. They do not read credential values or create runtime state.
|
or load a session. They resolve configured typed credential references as part
|
||||||
|
of the ordinary ConfigKit load but expose only their environment-variable names;
|
||||||
|
they do not create runtime state.
|
||||||
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
||||||
campaign selectors, resolves each named profile independently from one parsed
|
campaign selectors, resolves each named profile with an independent ConfigKit
|
||||||
root source set, and does not accept a separate `--profile` flag.
|
load, and does not accept a separate `--profile` flag.
|
||||||
|
|
||||||
Campaign selection is optional only when the resolved pipeline has no
|
Campaign selection is optional only when the resolved pipeline has no
|
||||||
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
||||||
@@ -59,11 +62,12 @@ then parses the campaign-owned party and expands concrete artifacts and any
|
|||||||
family publish rules before validation. `config validate` prints the resulting
|
family publish rules before validation. `config validate` prints the resulting
|
||||||
root, profile, and effective digest. `config show` emits the normalized
|
root, profile, and effective digest. `config show` emits the normalized
|
||||||
effective pipeline YAML, with defaults and concrete expansion included but
|
effective pipeline YAML, with defaults and concrete expansion included but
|
||||||
composition and family declarations omitted. `config sources` prints a stable
|
ConfigKit directives and family declarations omitted. This review projection is
|
||||||
|
not promised to be a reloadable authoring document. `config sources` prints a stable
|
||||||
source projection instead of effective values: root/import/profile/default
|
source projection instead of effective values: root/import/profile/default
|
||||||
ownership plus campaign/party and generated-family records. Canonical derived
|
ownership plus campaign/party and generated-family records. Canonical derived
|
||||||
players trace to the party; a legacy configured players file is explicitly
|
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||||
marked as a legacy player source. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
owns command syntax and output conventions.
|
owns command syntax and output conventions.
|
||||||
|
|
||||||
`config diff` compares normalized field values rather than YAML text or source
|
`config diff` compares normalized field values rather than YAML text or source
|
||||||
@@ -77,14 +81,15 @@ difference.
|
|||||||
### Migrating to the maintained bundle
|
### Migrating to the maintained bundle
|
||||||
|
|
||||||
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||||
as the complete copyable migration reference. Split stable pipeline settings
|
as the complete copyable migration reference. Keep production settings in the
|
||||||
into explicit additive imports, place production/testing differences in one
|
base, split them into explicit additive imports, and place testing differences
|
||||||
selected overlay, and retain a production `default_profile`. Convert campaign
|
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
|
||||||
rosters to [canonical party input](integrations/party.md), remove a separate
|
in the recommended one-definition files shown by that bundle. Convert campaign
|
||||||
`players_file`, then express character work as families. Inspect the result
|
rosters to [canonical party input](integrations/party.md), then express character
|
||||||
with `config validate`, `config show`, and `config sources`; use `config diff`
|
work as families. Inspect the result with `config validate`, `config show`,
|
||||||
to review profiles before running a session. Unversioned parties and their
|
`config sources`, and `config artifacts`; use `config diff default testing` to
|
||||||
`players_file` remain a clearly bounded legacy compatibility path.
|
review the overlay before running a session. Unversioned party documents are
|
||||||
|
rejected.
|
||||||
|
|
||||||
### Identity segments
|
### Identity segments
|
||||||
|
|
||||||
@@ -97,51 +102,51 @@ remote state with an unsafe legacy identity must be migrated before use.
|
|||||||
|
|
||||||
## Validation and Merge Rules
|
## Validation and Merge Rules
|
||||||
|
|
||||||
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
|
||||||
unknown fields or trailing documents fail load.
|
unknown fields, duplicate keys, and trailing documents fail load.
|
||||||
- A pipeline file may explicitly import additive YAML fragments through the
|
- A pipeline file may explicitly import additive YAML fragments through the
|
||||||
root-only `composition.imports` list. Imported files contribute fields to one
|
root-only `configkit.imports` list. Imported files contribute fields to one
|
||||||
logical pipeline document; they do not override fields supplied by the root
|
logical pipeline document; they do not override fields supplied by the root
|
||||||
or another import.
|
or another import.
|
||||||
- A root pipeline may declare named profiles. Exactly one profile is selected
|
- An omitted profile, an empty profile option, and the exact name `default`
|
||||||
by an option-aware caller or by `composition.default_profile`; a caller's
|
select the base configuration. Any other name selects
|
||||||
explicit selection takes precedence. Declaring profiles without either form
|
`profiles/<name>.yml` relative to the pipeline root.
|
||||||
of selection is an error.
|
- Campaign, session, and canonical-party documents are strict single-source
|
||||||
|
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
|
||||||
|
directives.
|
||||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||||
|
- Session and party inputs inspected before typed loading are limited to 8 MiB.
|
||||||
- Pipeline defaults are applied before validation.
|
- Pipeline defaults are applied before validation.
|
||||||
|
- Defaults apply only to absent fields. Explicit empty strings, zero values,
|
||||||
|
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
|
||||||
|
Narratio domain validation; they are not silently replaced by defaults.
|
||||||
|
Explicit null optional stage sections stay absent, while required sections
|
||||||
|
fail validation.
|
||||||
- Campaign and session identities must agree.
|
- Campaign and session identities must agree.
|
||||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||||
`party_file`) and the optional `spell_catalog_file` resolve from session
|
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||||
overrides when provided, otherwise from campaign defaults. An empty or
|
session. The optional `spell_catalog_file` also inherits a campaign value
|
||||||
omitted session spell-catalog value inherits the campaign value.
|
when omitted by the session.
|
||||||
- `party_file` is classified when pipeline and campaign configuration are
|
- `party_file` is parsed when pipeline and campaign configuration are combined.
|
||||||
combined. A versioned [canonical party](integrations/party.md) is
|
It must be a versioned [canonical party](integrations/party.md), is
|
||||||
campaign-owned, derives the players input internally, and forbids both a
|
campaign-owned and derives the players input internally. `players_file` and
|
||||||
separate `players_file` and a session `party_file` override. An unversioned
|
session `party_file` are unsupported configuration keys.
|
||||||
party remains a bounded legacy input and requires `players_file`; its normal
|
|
||||||
campaign/session overrides continue to apply.
|
|
||||||
- Exactly one audio mode must be configured in session input:
|
- Exactly one audio mode must be configured in session input:
|
||||||
- local (`audio_dir` or `audio_files`), or
|
- local (`audio_dir` or `audio_files`), or
|
||||||
- S3 (`audio_s3.prefix`).
|
- S3 (`audio_s3.prefix`).
|
||||||
|
|
||||||
### Pipeline composition
|
### Pipeline imports and profiles
|
||||||
|
|
||||||
Large pipeline configurations may be split into explicitly named fragments and
|
Large pipeline configurations may be split into explicitly named fragments and
|
||||||
may declare one overlay per selectable profile:
|
may use fixed-path profile overlays:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
composition:
|
configkit:
|
||||||
imports:
|
imports:
|
||||||
- config/storage.yml
|
- path: config/storage.yml
|
||||||
- config/integrations.yaml
|
- path: config/integrations.yaml
|
||||||
default_profile: production
|
|
||||||
profiles:
|
|
||||||
production:
|
|
||||||
overlay: profiles/production.yml
|
|
||||||
testing:
|
|
||||||
overlay: profiles/testing.yml
|
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
root: /usr/local/share/narratio/campaigns
|
root: /usr/local/share/narratio/campaigns
|
||||||
@@ -155,36 +160,29 @@ Imports are resolved relative to the directory containing the root pipeline
|
|||||||
file and are loaded in declaration order. Narratio does not scan directories or
|
file and are loaded in declaration order. Narratio does not scan directories or
|
||||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||||
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
||||||
import of the root pipeline itself are rejected. Only the root pipeline may
|
import of the root pipeline itself are rejected. Imported files cannot declare
|
||||||
contain `composition`; nested composition is rejected.
|
another `configkit` control section.
|
||||||
|
|
||||||
Composition is additive. A map may be extended by multiple files when every
|
Base merging is additive. A map may be extended by multiple files when every
|
||||||
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
||||||
more than once, even when the repeated values are identical. Conflict errors
|
more than once, even when the repeated values are identical. Conflict errors
|
||||||
name the full field path and every source that claimed it. The assembled YAML is
|
name the full field path and every source that claimed it. Narratio additionally
|
||||||
then decoded against the normal strict pipeline schema and defaults are applied
|
rejects an artifact or artifact-family key declared by more than one base file,
|
||||||
once.
|
even when those files contribute disjoint children.
|
||||||
|
|
||||||
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
|
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
|
||||||
control characters. If `profiles` is present, it must contain at least one
|
rules. Only the selected overlay is opened; a malformed unselected overlay does
|
||||||
entry and every entry must contain only an `overlay` path. An explicit profile
|
not affect a base or different-profile load. Maps merge recursively, overlay
|
||||||
selection overrides `default_profile`; unknown and explicitly empty selections
|
scalars replace base scalars, and overlay lists replace base lists completely.
|
||||||
fail. Narratio never selects the first profile implicitly and does not read a
|
Explicit `false`, zero, empty strings, empty containers, and null values remain
|
||||||
profile selection from the environment.
|
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
|
||||||
|
stack, import files, or declare another ConfigKit control section.
|
||||||
|
|
||||||
Every declared overlay is resolved relative to the root pipeline directory and
|
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
|
||||||
must satisfy the same confined regular-YAML-file rules as an import. Narratio
|
entry-level defaults and normalization, read-only file-local validation, and
|
||||||
parses every declared overlay even when it is not selected, then applies only
|
ordinary path resolution before computing a deterministic effective
|
||||||
the selected one. Maps merge recursively, overlay scalars replace base scalars,
|
|
||||||
and overlay lists replace base lists completely. Explicit `false`, zero, empty
|
|
||||||
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
|
|
||||||
kind changes are rejected. Profiles cannot inherit from or stack with other
|
|
||||||
profiles, and overlays cannot import files or declare profiles.
|
|
||||||
|
|
||||||
After composition, Narratio strictly decodes the result, applies centralized
|
|
||||||
defaults once, resolves ordinary paths, and computes a deterministic effective
|
|
||||||
configuration digest. The digest represents the normalized, secret-free
|
configuration digest. The digest represents the normalized, secret-free
|
||||||
runtime pipeline mapping; it excludes composition declarations, source
|
runtime pipeline mapping; it excludes ConfigKit directives, source
|
||||||
provenance, profile identity, and raw environment secret values. Equivalent
|
provenance, profile identity, and raw environment secret values. Equivalent
|
||||||
effective mappings therefore have the same digest regardless of how fields are
|
effective mappings therefore have the same digest regardless of how fields are
|
||||||
split among the root and imports.
|
split among the root and imports.
|
||||||
@@ -213,7 +211,6 @@ inputs:
|
|||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
players_file: ./players.yml
|
|
||||||
party_file: ./party.yml
|
party_file: ./party.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -228,15 +225,31 @@ inputs:
|
|||||||
## Secrets Handling
|
## Secrets Handling
|
||||||
|
|
||||||
- Do not place raw secrets in YAML.
|
- Do not place raw secrets in YAML.
|
||||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
- Credential fields contain environment-variable names, represented internally
|
||||||
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
as ConfigKit typed secret references (for example
|
||||||
environment-variable filename supplies one value; trailing CR/LF is removed.
|
`pipeline.notarius.llm_api_key: NOTARIUS_LLM_API_KEY`). A configured reference
|
||||||
|
must resolve to a nonempty environment value during load.
|
||||||
|
- Every surviving reference is resolved during ordinary pipeline loading,
|
||||||
|
regardless of whether its integration is disabled, outside the selected
|
||||||
|
stage range, or eligible for resume. Omit the field or set it to `null` when
|
||||||
|
that deployment should not require the credential.
|
||||||
|
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
|
||||||
|
flag is omitted, Narratio uses a present `secrets/` directory beside the
|
||||||
|
resolved root `pipeline.yml`; an absent conventional directory is ignored.
|
||||||
|
Explicit relative paths are resolved from the caller's working directory,
|
||||||
|
and a missing or empty explicit selection is an error.
|
||||||
|
- Each immediate regular file whose name is a valid environment-variable name
|
||||||
|
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
|
||||||
|
selected directory, rejects non-hidden symlink entries, and limits input to
|
||||||
|
1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
|
||||||
|
files, directories, and symlinks) are ignored and omitted from bootstrap
|
||||||
|
reports, but still count toward the directory entry limit.
|
||||||
- An existing process environment value takes precedence over a credential file.
|
- An existing process environment value takes precedence over a credential file.
|
||||||
- Credential directories and files must not be symlinks and must be regular,
|
- On POSIX, operators must provision the directory with no group/other access
|
||||||
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
(normally `0700`) and files with no group/other access (normally `0600`);
|
||||||
with no group/other access (normally `0700`) and files with no group/other
|
ConfigKit validates type and bounds but does not enforce permission modes.
|
||||||
access (normally `0600`).
|
- Bootstrap happens once per command, before any configuration load or adapter
|
||||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
construction. Safe reports contain filenames, paths, and statuses, never values.
|
||||||
|
|
||||||
## Publish Configuration Summary
|
## Publish Configuration Summary
|
||||||
|
|
||||||
@@ -259,9 +272,6 @@ publish:
|
|||||||
- source: narratio.artifact.session_recap
|
- source: narratio.artifact.session_recap
|
||||||
dest: artifacts/session_recap.md
|
dest: artifacts/session_recap.md
|
||||||
required: true
|
required: true
|
||||||
locks:
|
|
||||||
- source: narratio.artifact.session_recap
|
|
||||||
reason: manual post-publish edits
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
@@ -272,7 +282,8 @@ Rules:
|
|||||||
a rule names that source; the Notarius index and complete bundle are not
|
a rule names that source; the Notarius index and complete bundle are not
|
||||||
publish sources.
|
publish sources.
|
||||||
- `outputs[].required` defaults to `true`.
|
- `outputs[].required` defaults to `true`.
|
||||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
- publish locks are independent remote session objects managed with
|
||||||
|
`narratio session locks`; they are not pipeline configuration fields.
|
||||||
|
|
||||||
## Full Schema
|
## Full Schema
|
||||||
|
|
||||||
@@ -280,24 +291,21 @@ Rules:
|
|||||||
|
|
||||||
| Field | Type | Required | Default / Rule |
|
| Field | Type | Required | Default / Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||||
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
|
|
||||||
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
|
|
||||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
|
||||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||||
| `pipeline.storage.s3.region` | string | No | empty |
|
| `pipeline.storage.s3.region` | string | No | empty |
|
||||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
|
||||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
|
||||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||||
@@ -306,9 +314,6 @@ Rules:
|
|||||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||||
| `pipeline.publish.locks[]` | list | No | empty |
|
|
||||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
|
||||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
|
||||||
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||||
| `pipeline.whisperx.language` | string | No | `en` |
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||||
@@ -326,7 +331,7 @@ Rules:
|
|||||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||||
| `pipeline.audita.binary` | string | No | `audita` |
|
| `pipeline.audita.binary` | string | No | `audita` |
|
||||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
| `pipeline.audita.llm_api_key` | secret reference | No | unset |
|
||||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||||
| `pipeline.audita.base_url` | string | No | empty |
|
| `pipeline.audita.base_url` | string | No | empty |
|
||||||
| `pipeline.audita.model` | string | No | empty |
|
| `pipeline.audita.model` | string | No | empty |
|
||||||
@@ -354,10 +359,14 @@ Rules:
|
|||||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||||
|
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
|
||||||
|
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
|
||||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
|
||||||
|
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
|
||||||
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
||||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||||
| `pipeline.render.enabled` | bool | No | `true` |
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
@@ -374,6 +383,73 @@ Rules:
|
|||||||
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
||||||
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||||
|
|
||||||
|
### Notarius Configuration Profile
|
||||||
|
|
||||||
|
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
|
||||||
|
configuration profile when extraction executes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: downstream-economy
|
||||||
|
```
|
||||||
|
|
||||||
|
When the field is omitted or `null`, Narratio supplies no downstream profile
|
||||||
|
selector and preserves Notarius's existing configuration-selection behavior.
|
||||||
|
When present, Narratio forwards the exact string to Notarius, including an
|
||||||
|
empty string or `default`, which explicitly select Notarius's composed base
|
||||||
|
configuration. Notarius owns selector interpretation and validation; Narratio
|
||||||
|
does not restrict profile names, resolve profile files, or rewrite the value.
|
||||||
|
|
||||||
|
This selection is independent of Narratio's own `--profile` selection. A
|
||||||
|
Narratio overlay may explicitly select a differently named Notarius overlay,
|
||||||
|
but Narratio never forwards its own profile name automatically. It is also
|
||||||
|
independent of `pipeline.notarius.llm_api_key`: an overlay may change either or
|
||||||
|
both fields according to the deployed Notarius configuration. See the
|
||||||
|
[Notarius integration contract](./integrations/notarius.md) for invocation and
|
||||||
|
failure behavior.
|
||||||
|
|
||||||
|
### Notarius Credential
|
||||||
|
|
||||||
|
`pipeline.notarius.llm_api_key` optionally names the one environment variable
|
||||||
|
to forward to the Notarius child process:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
llm_api_key: OPENROUTER_API_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
The field contains the variable name, not a credential value. That name must
|
||||||
|
match `api_key_env` in the Notarius/PromptKit profile selected by the deployed
|
||||||
|
Notarius pipeline; Narratio does not infer or remap provider names. Omission or
|
||||||
|
`null` forwards nothing. A non-null selected reference must resolve at pipeline
|
||||||
|
load time through the process environment or existing secrets-directory
|
||||||
|
bootstrap, even when Notarius is disabled, extract is outside the requested
|
||||||
|
range, or a completed extraction can be reused.
|
||||||
|
|
||||||
|
The Notarius child receives only Narratio's minimal execution environment plus
|
||||||
|
this exact selected name and loaded value. Other credentials are not forwarded.
|
||||||
|
Changing the value, name, presence, or absence does not invalidate extract's
|
||||||
|
semantic resume evidence. The public name remains part of safe effective
|
||||||
|
inspection, so renaming it may appear in `config diff` and change the effective
|
||||||
|
configuration digest; rotating only its value does neither.
|
||||||
|
|
||||||
|
See the [Notarius integration contract](./integrations/notarius.md) for the
|
||||||
|
delivery and redaction boundary.
|
||||||
|
|
||||||
|
### Notarius Cache Controls
|
||||||
|
|
||||||
|
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
|
||||||
|
extraction passes `--resume`; the deployed Notarius configuration must already
|
||||||
|
enable checkpoint recording and should select an explicit persistent checkpoint
|
||||||
|
root. Narratio does not inspect or generate that configuration.
|
||||||
|
|
||||||
|
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
|
||||||
|
configured mode in control. A nonempty value must be `auto`, `bypass`, or
|
||||||
|
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
|
||||||
|
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
|
||||||
|
Neither field changes Narratio's semantic extraction fingerprint, so changing
|
||||||
|
them alone does not rerun an otherwise complete durable extraction.
|
||||||
|
|
||||||
### Notarius Reference Bindings
|
### Notarius Reference Bindings
|
||||||
|
|
||||||
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
||||||
@@ -403,8 +479,8 @@ It validates only selector structure and the prepared source vocabulary;
|
|||||||
Notarius owns target-slot declarations and media compatibility.
|
Notarius owns target-slot declarations and media compatibility.
|
||||||
|
|
||||||
Before extraction, Narratio resolves every binding from the current prepared
|
Before extraction, Narratio resolves every binding from the current prepared
|
||||||
session manifest and streams it into a verified invocation-local snapshot whose
|
session manifest and streams it into a verified, content-addressed session
|
||||||
absolute path is passed to Notarius. Missing, unsafe, empty,
|
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
|
||||||
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
||||||
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
||||||
part of extraction fingerprint and resume identity. See the
|
part of extraction fingerprint and resume identity. See the
|
||||||
@@ -439,7 +515,7 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
|||||||
| Field | Type | Required | Rule |
|
| Field | Type | Required | Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | bool | No | `false` if omitted |
|
| `enabled` | bool | No | `false` if omitted |
|
||||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; configured graph must be acyclic |
|
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
|
||||||
| `render_debug` | bool | No | per-artifact override |
|
| `render_debug` | bool | No | per-artifact override |
|
||||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||||
| `profile_id` | string | No | empty |
|
| `profile_id` | string | No | empty |
|
||||||
@@ -453,8 +529,8 @@ Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium re
|
|||||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||||
those targets over `depends_on`: a current prerequisite is reused, while a
|
those targets over the effective dependency graph: a current prerequisite is reused, while a
|
||||||
stale, missing, failed, or legacy prerequisite is rebuilt before its dependent.
|
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||||
may require rebuilding must therefore have valid executable fields. This
|
may require rebuilding must therefore have valid executable fields. This
|
||||||
override affects analyze planning only; publish uses the list only to filter
|
override affects analyze planning only; publish uses the list only to filter
|
||||||
@@ -471,12 +547,22 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||||
does not provide adapter-specific input passthrough fields.
|
does not provide adapter-specific input passthrough fields.
|
||||||
|
|
||||||
|
Each `narratio.artifact.<name>` input source automatically contributes an
|
||||||
|
effective dependency on that configured artifact. Narratio unions these inferred
|
||||||
|
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
|
||||||
|
result during final configuration resolution. Authors use `depends_on` only for
|
||||||
|
ordering constraints that are not already expressed by an artifact input.
|
||||||
|
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
|
to inspect keys, definition sources, outputs, and effective dependencies without
|
||||||
|
loading a session or creating runtime state.
|
||||||
|
|
||||||
### Scriptorium Artifact Families
|
### Scriptorium Artifact Families
|
||||||
|
|
||||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||||
for every canonical campaign character. Configuration resolution expands each
|
for every canonical campaign character. Configuration resolution expands each
|
||||||
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
|
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
|
||||||
planning or Scriptorium invocation. A legacy party cannot be used for a family.
|
planning or Scriptorium invocation. Party parsing rejects unsupported document
|
||||||
|
schemas before family expansion.
|
||||||
|
|
||||||
For each `pipeline.scriptorium.artifact_families.<name>`:
|
For each `pipeline.scriptorium.artifact_families.<name>`:
|
||||||
|
|
||||||
@@ -501,9 +587,10 @@ Their resolved values are strings. A member variable may not reuse a static
|
|||||||
other Scriptorium artifact.
|
other Scriptorium artifact.
|
||||||
|
|
||||||
Within a family only, an input source may use
|
Within a family only, an input source may use
|
||||||
`narratio.member_artifact.<family>`. The referenced family must be named in
|
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
|
||||||
that family's `member_dependencies`; resolution rewrites the source to the
|
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
|
||||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source.
|
infers the concrete member dependency. Use `member_dependencies` only for
|
||||||
|
additional same-character ordering constraints that are not already inputs.
|
||||||
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
||||||
or Scriptorium.
|
or Scriptorium.
|
||||||
|
|
||||||
@@ -524,7 +611,6 @@ integration.
|
|||||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||||
| `inputs.players_file` | string | Conditional | required only with an unversioned legacy `party_file`; forbidden for a canonical party |
|
|
||||||
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
|
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
|
||||||
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
|
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
|
||||||
|
|
||||||
@@ -540,8 +626,6 @@ integration.
|
|||||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||||
| `inputs.players_file` | string | No | legacy-party override; forbidden for a canonical party |
|
|
||||||
| `inputs.party_file` | string | No | legacy-party override; forbidden for a canonical campaign party |
|
|
||||||
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
|
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
|
||||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||||
|
|||||||
@@ -10,6 +10,11 @@ polished transcripts and generated artifacts. Start with the
|
|||||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||||
|
|
||||||
|
## Go Prerequisite
|
||||||
|
|
||||||
|
Maintained development and verification require Go 1.26. Use the official Go
|
||||||
|
1.26 toolchain before running repository commands.
|
||||||
|
|
||||||
## What To Read
|
## What To Read
|
||||||
|
|
||||||
| When working on | Read | Why |
|
| When working on | Read | Why |
|
||||||
@@ -53,7 +58,7 @@ in the [Release Procedure](release.md).
|
|||||||
|
|
||||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||||
for macOS and Windows, but do not provide native macOS or Windows execution.
|
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
|
||||||
Those cross-builds establish compilation only, not platform-equivalent runtime
|
target. The macOS cross-build establishes compilation only, not native runtime
|
||||||
evidence. Add native checks only when official runner labels and successful
|
evidence. Add a native check only when an official runner label and successful
|
||||||
native-run evidence are available.
|
native-run evidence are available.
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ Construction fails for invalid static config values, including:
|
|||||||
|
|
||||||
Run fails for:
|
Run fails for:
|
||||||
- missing required request paths;
|
- missing required request paths;
|
||||||
- missing required credential env var when configured (`llm_api_key_env`);
|
|
||||||
- subprocess execution failure;
|
- subprocess execution failure;
|
||||||
- invalid processed transcript JSON (`segments` array required);
|
- invalid processed transcript JSON (`segments` array required);
|
||||||
- invalid report JSON when reporting is enabled.
|
- invalid report JSON when reporting is enabled.
|
||||||
@@ -53,6 +52,9 @@ Failure results still include output/log/config/exit metadata for diagnostics.
|
|||||||
|
|
||||||
## Deterministic Behavior
|
## Deterministic Behavior
|
||||||
- CLI args are built from runner config + request in a fixed order.
|
- CLI args are built from runner config + request in a fixed order.
|
||||||
|
- A configured typed `llm_api_key` reference is resolved during pipeline load;
|
||||||
|
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
|
||||||
|
override and is never re-read from the parent environment.
|
||||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||||
|
|
||||||
|
|||||||
@@ -7,14 +7,14 @@ lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
|||||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||||
|
|
||||||
Canonical Notarius v0.6.0 references:
|
Canonical Notarius v0.7.0 references:
|
||||||
|
|
||||||
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/cli.md)
|
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
|
||||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/subprocess.md)
|
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
|
||||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/dnd-pipeline.md)
|
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
|
||||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/run-result.md)
|
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
|
||||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/json-output.md)
|
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
|
||||||
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
||||||
|
|
||||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||||
@@ -25,19 +25,38 @@ duplicate the complete schemas.
|
|||||||
|
|
||||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||||
configuration path, input path, output directory, and working directory to
|
configuration path, input path, output directory, and working directory to
|
||||||
absolute paths. Narratio requires the Notarius v0.6.0 CLI contract when
|
absolute paths. Configured references require support for the `--reference`
|
||||||
references are configured and invokes each binding as a separate argument
|
contract introduced in Notarius v0.6.0. Cache controls require support for the
|
||||||
before `--json`:
|
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
|
||||||
|
cache integration baseline, not an unconditional minimum version; Narratio
|
||||||
|
does not impose a runtime version gate. Each reference binding is a separate
|
||||||
|
argument before `--json`. When `pipeline.notarius.config_profile` is present,
|
||||||
|
its exact value is another argument pair before `--json`:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--reference <selector>=<verified_snapshot_path>]... --json
|
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
|
||||||
```
|
```
|
||||||
|
|
||||||
Reference paths are absolute invocation-local snapshots streamed from the
|
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
|
||||||
manifest-verified canonical files prepared inside the current Narratio session
|
option requires a compatible Notarius executable, but deployments that omit the
|
||||||
workspace. Narratio verifies snapshot checksum and size before and after the
|
field acquire no additional version requirement. Omitted or null configuration
|
||||||
subprocess, and passes only configured bindings, ordered lexically by normalized
|
adds no flag. Supplied empty, `default`, and named selectors are forwarded
|
||||||
selector, as direct argument-vector entries without shell interpretation. A CLI
|
without normalization so Notarius can apply its own ConfigKit semantics. This
|
||||||
|
option selects a Notarius configuration overlay, not a PromptKit profile through
|
||||||
|
`--llm-profile`, and Narratio never substitutes its own selected profile name.
|
||||||
|
A missing or invalid downstream overlay is an ordinary Notarius subprocess
|
||||||
|
failure: Narratio does not retry without the selector or fall back to another
|
||||||
|
profile.
|
||||||
|
|
||||||
|
Reference paths are absolute stable snapshots streamed from manifest-verified
|
||||||
|
canonical files prepared inside the current Narratio session workspace. They
|
||||||
|
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
|
||||||
|
full lowercase content digest. Narratio verifies snapshot checksum and size
|
||||||
|
before and after the subprocess, reconstructs missing or corrupt regular files
|
||||||
|
from a still-valid prepared source, and passes only configured bindings,
|
||||||
|
ordered lexically by normalized selector, as direct argument-vector entries
|
||||||
|
without shell interpretation. Symlinks, directories, other non-regular
|
||||||
|
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
|
||||||
binding takes precedence over a matching external path in Notarius
|
binding takes precedence over a matching external path in Notarius
|
||||||
configuration. Narratio never emits `--without-reference`.
|
configuration. Narratio never emits `--without-reference`.
|
||||||
|
|
||||||
@@ -75,9 +94,41 @@ It does not pass a Narratio session ID or run `notarius config validate`
|
|||||||
automatically; the configured working directory and Narratio's minimal child
|
automatically; the configured working directory and Narratio's minimal child
|
||||||
environment apply to the subprocess.
|
environment apply to the subprocess.
|
||||||
|
|
||||||
|
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
|
||||||
|
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
|
||||||
|
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
|
||||||
|
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
|
||||||
|
Narratio neither reads nor synthesizes its configuration. Use explicit
|
||||||
|
persistent roots in Notarius when reuse must span subprocess invocations.
|
||||||
|
Changing a reference path/digest or other upstream identity input correctly
|
||||||
|
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
|
||||||
|
replacement for the active source; it does not delete unrelated plans or
|
||||||
|
checkpoint state.
|
||||||
|
|
||||||
|
When `pipeline.notarius.llm_api_key` is absent or null, Narratio adds no
|
||||||
|
credential to that environment. When configured, the field names an environment
|
||||||
|
variable whose already-loaded nonempty value is forwarded under that exact
|
||||||
|
name. Operators must use the same name as the `api_key_env` in the selected
|
||||||
|
Notarius/PromptKit profile. Narratio does not inspect the Notarius configuration
|
||||||
|
to infer a provider or destination name, and it does not forward unrelated
|
||||||
|
credentials from its parent environment or secrets directory.
|
||||||
|
|
||||||
|
The selected name is registered with Narratio's shared streaming redactor.
|
||||||
|
Credential bytes written by the child to captured stdout, captured stderr, or
|
||||||
|
stderr-derived failure diagnostics are replaced with `<redacted>`, including
|
||||||
|
custom names not known to the built-in sensitive-name list. Narratio does not
|
||||||
|
serialize the credential into arguments, stage metadata, manifests, or
|
||||||
|
generated configuration.
|
||||||
|
|
||||||
|
This captured-stream protection is not a filesystem sandbox. Notarius remains
|
||||||
|
responsible for keeping credentials out of files it writes directly, including
|
||||||
|
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
|
||||||
|
arbitrary downstream files and cannot promise redaction of transformed or
|
||||||
|
encoded credentials.
|
||||||
|
|
||||||
## Accepted Result
|
## Accepted Result
|
||||||
|
|
||||||
Narratio's supported invocation baseline is Notarius v0.6.0. The accepted
|
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
|
||||||
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
||||||
receipt or ten-lane output contract. The receipt
|
receipt or ten-lane output contract. The receipt
|
||||||
must identify the configured pipeline, and its `index_file` must be exactly
|
must identify the configured pipeline, and its `index_file` must be exactly
|
||||||
@@ -102,6 +153,8 @@ version, and, when configured, module key. Missing, duplicate, rejected, or
|
|||||||
incompatible required lanes fail extraction even if Notarius exited zero. A
|
incompatible required lanes fail extraction even if Notarius exited zero. A
|
||||||
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
||||||
fails extraction.
|
fails extraction.
|
||||||
|
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
|
||||||
|
preserves that valid summary rather than treating it as a missing field.
|
||||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||||
selectable Narratio sources.
|
selectable Narratio sources.
|
||||||
|
|
||||||
@@ -130,3 +183,18 @@ Configuration fields and defaults are in [Configuration](../config.md).
|
|||||||
Operator paths, rerun procedures, and bundle retention are in
|
Operator paths, rerun procedures, and bundle retention are in
|
||||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||||
for failure recovery.
|
for failure recovery.
|
||||||
|
|
||||||
|
## Opt-in Integration Test
|
||||||
|
|
||||||
|
The production-boundary cache test requires an absolute path to a locally
|
||||||
|
built Notarius v0.7.0-compatible executable. It uses only a local
|
||||||
|
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
|
||||||
|
GOWORK=off go test ./internal/stage \
|
||||||
|
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
|
||||||
|
```
|
||||||
|
|
||||||
|
When the variable is unset, the normal test suite skips this test and never
|
||||||
|
searches `PATH`, a sibling checkout, the network, or a real provider.
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# Canonical Party Input
|
# Canonical Party Input
|
||||||
|
|
||||||
`party.yml` is a campaign-owned roster input. Narratio recognizes the
|
`party.yml` is a campaign-owned roster input. It must be a single, versioned
|
||||||
versioned `narratio.party.v1` document below when it resolves a pipeline,
|
`narratio.party.v1` document when Narratio resolves a pipeline, campaign, and
|
||||||
campaign, and session together.
|
session together.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
schema_version: narratio.party.v1
|
schema_version: narratio.party.v1
|
||||||
@@ -34,19 +34,17 @@ non-empty, trimmed display strings without control characters. Character names
|
|||||||
and aliases must be unique across the full roster under Unicode-aware
|
and aliases must be unique across the full roster under Unicode-aware
|
||||||
case-insensitive comparison; player names may repeat.
|
case-insensitive comparison; player names may repeat.
|
||||||
|
|
||||||
The document has exactly one YAML document and accepts no unknown fields. A
|
The document has exactly one YAML document and accepts no unknown fields.
|
||||||
wrong or malformed `schema_version` is an error.
|
`schema_version` is required: a missing, wrong, or malformed value is an error.
|
||||||
|
|
||||||
## Legacy migration boundary
|
## Migration
|
||||||
|
|
||||||
An unversioned party input remains supported only as opaque legacy reference
|
Unversioned party inputs are rejected. Replace each legacy party document with
|
||||||
material while campaigns migrate. It requires a separate `players_file` and
|
the versioned structure above, move each roster entry under `characters`, and
|
||||||
retains the existing session override behavior. It cannot be mixed with a
|
remove separate player and session-party settings: those keys are unsupported.
|
||||||
canonical party: canonical campaigns must omit `players_file`, and sessions
|
|
||||||
must not override their party or players inputs.
|
|
||||||
|
|
||||||
Use the canonical document for new campaigns. The configuration rules and
|
The configuration rules and source-relative path behavior are defined in the
|
||||||
source-relative path behavior are defined in the [Configuration Reference](../config.md).
|
[Configuration Reference](../config.md).
|
||||||
|
|
||||||
## Derived players document
|
## Derived players document
|
||||||
|
|
||||||
|
|||||||
@@ -17,11 +17,20 @@ Narratio sends an HTTP `POST` to the configured transcription URL using
|
|||||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||||
not currently require a more specific response schema at this boundary.
|
not currently require a more specific response schema at this boundary.
|
||||||
|
|
||||||
The transcription URL must be an absolute `http` or `https` URL. The audio body
|
The transcription URL must be an absolute `http` or `https` URL. Each
|
||||||
is streamed through a fresh multipart writer for every attempt, so its memory
|
application attempt creates one bounded multipart framing and streams the audio
|
||||||
use is bounded by the transport buffer rather than by the complete audio file.
|
from an open file between that framing; it does not buffer the complete audio
|
||||||
|
or spool a multipart copy. The file field precedes the language field.
|
||||||
WhisperX response acquisition is capped at 10 MiB.
|
WhisperX response acquisition is capped at 10 MiB.
|
||||||
|
|
||||||
|
The initial request body and any transport-requested replay use the same
|
||||||
|
multipart boundary and framing. Every body opens an independent audio handle
|
||||||
|
and replay starts at the beginning. Narratio closes all bodies created for the
|
||||||
|
attempt on response, failure, timeout, or cancellation, including when a
|
||||||
|
transport abandons an earlier body. Genuine source read/close failures remain
|
||||||
|
request failures; deliberate early closure of an abandoned original does not
|
||||||
|
poison a valid replay.
|
||||||
|
|
||||||
## Request And Result Contract
|
## Request And Result Contract
|
||||||
|
|
||||||
Each adapter request identifies a speaker, a readable audio file, and the
|
Each adapter request identifies a speaker, a readable audio file, and the
|
||||||
@@ -43,6 +52,11 @@ transcript output.
|
|||||||
- Narratio waits the configured retry delay between attempts and aborts that
|
- Narratio waits the configured retry delay between attempts and aborts that
|
||||||
wait when the parent context is canceled.
|
wait when the parent context is canceled.
|
||||||
|
|
||||||
|
Transport replay occurs inside one application attempt and does not consume an
|
||||||
|
entry from the configured retry budget or increment the reported attempt
|
||||||
|
count. The standard HTTP redirect policy remains in effect; body replay can
|
||||||
|
also support body-preserving redirects.
|
||||||
|
|
||||||
## Validation And Failure Semantics
|
## Validation And Failure Semantics
|
||||||
|
|
||||||
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
||||||
@@ -55,12 +69,19 @@ failure to install the output causes the transcription to fail. Errors include
|
|||||||
attempt context, and the result retains attempts, final status when available,
|
attempt context, and the result retains attempts, final status when available,
|
||||||
and elapsed duration for diagnostics.
|
and elapsed duration for diagnostics.
|
||||||
|
|
||||||
|
Replay makes a fresh request body available to the HTTP transport; it does not
|
||||||
|
provide an exactly-once server guarantee and does not prove recovery from every
|
||||||
|
proxy or protocol-level failure. A server may have received work before the
|
||||||
|
client learns that a request failed. WhisperX operations should therefore be
|
||||||
|
safe to retry for the same input when practical.
|
||||||
|
|
||||||
## Determinism And Concurrency
|
## Determinism And Concurrency
|
||||||
|
|
||||||
Each audio request has stable multipart field names, and successful bytes are
|
Each audio request has stable multipart field names, and successful bytes are
|
||||||
installed atomically. The transcribe stage may process speaker files in
|
installed atomically. The transcribe stage may process speaker files in
|
||||||
parallel, bounded by the configured concurrency. It records results in stable
|
parallel, bounded by the configured concurrency. Each validated speaker is
|
||||||
speaker order after all work completes; any speaker failure fails the stage.
|
checkpointed independently; any speaker failure still fails the aggregate
|
||||||
|
stage, while committed earlier speakers remain reusable on an ordinary retry.
|
||||||
|
|
||||||
## Related Canonical Docs
|
## Related Canonical Docs
|
||||||
|
|
||||||
|
|||||||
@@ -35,8 +35,9 @@ Adapters do not own:
|
|||||||
|
|
||||||
## Default Wiring
|
## Default Wiring
|
||||||
|
|
||||||
`internal/app/runner.go` initializes default adapters when not injected and
|
The application composes default adapters per invocation when they were not
|
||||||
only when the selected execution plan needs them:
|
injected through the stage environment, and only when the selected execution
|
||||||
|
plan needs them:
|
||||||
|
|
||||||
- WhisperX HTTP client for `transcribe`.
|
- WhisperX HTTP client for `transcribe`.
|
||||||
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
|
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
|
||||||
@@ -49,8 +50,10 @@ only when the selected execution plan needs them:
|
|||||||
Remote publish locks are loaded only for a selected, enabled publish that
|
Remote publish locks are loaded only for a selected, enabled publish that
|
||||||
uploads a run. Shared session lifecycle setup still applies to every selected
|
uploads a run. Shared session lifecycle setup still applies to every selected
|
||||||
range, but an unselected integration is neither initialized nor validated by
|
range, but an unselected integration is neither initialized nor validated by
|
||||||
runner composition. Each selected stage retains its own fail-fast configuration
|
application-owned composition. Constructor, clock, randomness, remote-lock,
|
||||||
and input validation.
|
and post-publish deletion boundaries are scoped to the invocation; stages still
|
||||||
|
receive their collaborators through the stage environment. Each selected stage
|
||||||
|
retains its own fail-fast configuration and input validation.
|
||||||
|
|
||||||
`session plan` is outside production adapter composition. It performs
|
`session plan` is outside production adapter composition. It performs
|
||||||
resume validation and models selected transitions against cloned manifest
|
resume validation and models selected transitions against cloned manifest
|
||||||
@@ -62,31 +65,44 @@ Notarius is composed only when extraction is enabled; the extract stage owns
|
|||||||
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
||||||
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
||||||
owner of serializing them as repeated `--reference` arguments before `--json`.
|
owner of serializing them as repeated `--reference` arguments before `--json`.
|
||||||
|
It also serializes an effective resume request as `--resume` and a nonempty
|
||||||
|
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
|
||||||
|
adapter, resolves force precedence before constructing the request.
|
||||||
|
Its request may also carry one already-resolved opaque ConfigKit secret
|
||||||
|
reference. Only the Notarius adapter reads that value, rejects an unresolved or
|
||||||
|
empty programmatic reference, and maps the exact configured name/value into the
|
||||||
|
shared subprocess override and sensitive-name fields. It never rereads the
|
||||||
|
parent environment. A nil reference preserves the previous invocation.
|
||||||
|
|
||||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
Application-owned object-store construction loads configured filesystem
|
||||||
configured filesystem secrets before adapter initialization.
|
secrets before invoking the invocation's storage factory. Production commands
|
||||||
|
use the concrete storage adapter constructor; focused tests can supply an
|
||||||
|
in-memory backend without changing package state.
|
||||||
|
|
||||||
## Failure Semantics
|
## Failure Semantics
|
||||||
|
|
||||||
- Constructor errors fail stage execution setup early.
|
- Constructor errors fail stage execution setup early.
|
||||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||||
- Shared subprocess execution starts an owned process group on Linux/macOS or a
|
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||||
owned tree before returning. After a natural leader exit, Unix checks for
|
an owned process group on Linux and macOS. Every terminal path disposes of
|
||||||
remaining group members and uses bounded graceful then forceful termination;
|
that owned tree before returning. After a natural leader exit, the runner checks
|
||||||
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
for remaining group members and uses bounded graceful then forceful termination.
|
||||||
|
Cancellation, deadlines, and
|
||||||
diagnostic limits use the same terminal disposal path without losing their
|
diagnostic limits use the same terminal disposal path without losing their
|
||||||
original result classification. Child environments contain only the execution
|
original result classification. Child environments contain only the execution
|
||||||
baseline and adapter-specified values; configured credentials are explicit
|
baseline and adapter-specified values; configured credentials are resolved
|
||||||
sensitive values. Stdout and stderr are redacted while streaming into separate
|
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
|
||||||
|
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
|
||||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||||
departed leader's descendant. Unsupported platforms reject owned command
|
departed leader's descendant. Unsupported platforms reject owned command
|
||||||
execution.
|
execution.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
- Composition: `internal/app/runner_composition.go`,
|
||||||
|
`internal/app/runner_dependencies.go`, `internal/app/object_store.go`
|
||||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||||
- `internal/adapters/whisperx/http_test.go`
|
- `internal/adapters/whisperx/http_test.go`
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ The internal registry recognizes these stable built-in source IDs:
|
|||||||
- `narratio.transcript.final_trimmed_markdown`
|
- `narratio.transcript.final_trimmed_markdown`
|
||||||
- `narratio.bounds.session`
|
- `narratio.bounds.session`
|
||||||
|
|
||||||
Registry entries bind each ID to its producer, output kind, canonical fallback,
|
Registry entries bind each ID to its producer, output kind, and content
|
||||||
and content validator. The focused stage documents own their input/output flow;
|
validator. The focused stage documents own their input/output flow;
|
||||||
[Configuration](../config.md) owns where operators may select these IDs.
|
[Configuration](../config.md) owns where operators may select these IDs.
|
||||||
|
|
||||||
## Configured, Extraction, And Previous-Session Sources
|
## Configured, Extraction, And Previous-Session Sources
|
||||||
@@ -58,10 +58,12 @@ Configured definitions are always registered. Without an explicit selection,
|
|||||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||||
exact named configured definitions become the effective set for that invocation,
|
exact named configured definitions become the effective set for that invocation,
|
||||||
regardless of their `enabled` value. The effective-set resolver itself does not
|
regardless of their `enabled` value. The effective-set resolver itself does not
|
||||||
expand dependencies; the analyze work planner closes those targets over their
|
expand dependencies. Configuration has already materialized the sorted union of
|
||||||
configured prerequisite graph. Availability is separate from executability.
|
explicit ordering edges and configured-artifact input edges, and the analyze
|
||||||
Configuration may normalize a family selection into its concrete generated
|
work planner closes targets over that one concrete prerequisite graph.
|
||||||
members before this resolver runs. The effective set retains optional family
|
Availability is separate from executability. Configuration may normalize a
|
||||||
|
family selection into its concrete generated members before this resolver runs.
|
||||||
|
The effective set retains optional family
|
||||||
and character origin metadata, but its keys, catalog sources, and runtime
|
and character origin metadata, but its keys, catalog sources, and runtime
|
||||||
lookups remain concrete configured-artifact identities.
|
lookups remain concrete configured-artifact identities.
|
||||||
Family publish policies are likewise expanded into ordinary configured-source
|
Family publish policies are likewise expanded into ordinary configured-source
|
||||||
@@ -69,8 +71,9 @@ publish rules during configuration resolution.
|
|||||||
Configured outputs, including non-executable prerequisites, become available
|
Configured outputs, including non-executable prerequisites, become available
|
||||||
only when the versioned analyze state identifies a current result whose source,
|
only when the versioned analyze state identifies a current result whose source,
|
||||||
contract, canonical configured path, size, and checksum match a confined
|
contract, canonical configured path, size, and checksum match a confined
|
||||||
no-follow regular file. An incidental canonical file and a legacy aggregate
|
no-follow regular file. An incidental canonical file is unavailable, and a
|
||||||
analyze output are unavailable.
|
manifest with an unsupported aggregate-only representation is rejected before
|
||||||
|
catalog construction.
|
||||||
Extraction entries are registered from configuration and become available only
|
Extraction entries are registered from configuration and become available only
|
||||||
after compatible extraction evidence is hydrated.
|
after compatible extraction evidence is hydrated.
|
||||||
|
|
||||||
@@ -91,8 +94,13 @@ Current provenance values:
|
|||||||
|
|
||||||
Built-ins:
|
Built-ins:
|
||||||
|
|
||||||
1. manifest producer outputs (when present)
|
- resolve only from the current manifest's succeeded registered producer;
|
||||||
2. canonical session-path fallback
|
- require exactly one output with the registered source ID and output kind, a
|
||||||
|
safe session-rooted path, producer run ID, canonical checksum, and valid
|
||||||
|
content; and
|
||||||
|
- verify the registered checksum against the durable bytes. Incidental files,
|
||||||
|
canonical filenames, duplicate outputs, failed/stale producer records, and
|
||||||
|
malformed evidence are unavailable.
|
||||||
|
|
||||||
Configured sources (`narratio.artifact.*`):
|
Configured sources (`narratio.artifact.*`):
|
||||||
|
|
||||||
@@ -103,7 +111,7 @@ Configured sources (`narratio.artifact.*`):
|
|||||||
record for the exact configured key and source ID, a complete contract, the
|
record for the exact configured key and source ID, a complete contract, the
|
||||||
configured canonical relative path, positive stored size, and stored
|
configured canonical relative path, positive stored size, and stored
|
||||||
checksum matching bytes read from a confined no-follow regular file; and
|
checksum matching bytes read from a confined no-follow regular file; and
|
||||||
- treat non-current statuses, legacy or malformed records, removed keys,
|
- treat non-current statuses, malformed records, removed keys,
|
||||||
unsafe or missing files, and size/checksum mismatches as unavailable without
|
unsafe or missing files, and size/checksum mismatches as unavailable without
|
||||||
rewriting manifest state. Catalog construction iterates current
|
rewriting manifest state. Catalog construction iterates current
|
||||||
configuration, so removed or renamed records are not advertised.
|
configuration, so removed or renamed records are not advertised.
|
||||||
@@ -135,9 +143,12 @@ Extraction sources (`narratio.extraction.*`):
|
|||||||
|
|
||||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
|
|
||||||
- resolve only from local `previous/` cache state;
|
- resolve only from one matching `previous_artifact` input record in the
|
||||||
- prefer manifest-backed previous-input paths;
|
current manifest, including its exact source ID and checksum;
|
||||||
- fallback to existing previous-cache filesystem paths.
|
- are hydrated only from the unique `published_output` declaration for their
|
||||||
|
canonical source in the selected remote commit; and
|
||||||
|
- never infer a path from a previous manifest, current configuration, basename,
|
||||||
|
or existing cache file.
|
||||||
|
|
||||||
Source absence is evaluated by the consuming artifact input. An optional input
|
Source absence is evaluated by the consuming artifact input. An optional input
|
||||||
is omitted from that invocation; a required input fails resolution. This is
|
is omitted from that invocation; a required input fails resolution. This is
|
||||||
@@ -165,25 +176,34 @@ Validation by content type:
|
|||||||
Artifacts package owns shared remote current-state loading mechanics used by
|
Artifacts package owns shared remote current-state loading mechanics used by
|
||||||
restore, status and validation checks, and previous-cache planning.
|
restore, status and validation checks, and previous-cache planning.
|
||||||
|
|
||||||
|
The selected session manifest is accepted only with
|
||||||
|
`schema_version: narratio.session-manifest.v1` and complete stage identity.
|
||||||
|
An unsupported local or downloaded manifest must be replaced by regeneration,
|
||||||
|
verified restore, or republication; artifact resolution never fills in missing
|
||||||
|
authority markers.
|
||||||
|
|
||||||
For a new-protocol current state, the pointer-selected immutable commit is the
|
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||||
complete restore authority. Callers receive its declared object identities and
|
complete restore authority. Callers receive its declared object identities and
|
||||||
must not supplement them by listing mutable session prefixes. The legacy reader
|
must not supplement them by listing mutable session prefixes.
|
||||||
is intentionally separate and remains migration-only support.
|
|
||||||
|
A `published_output` declaration must use a canonical publish-compatible source
|
||||||
|
ID and an exact `runs/{run_id}/published/{destination}` key in the same session.
|
||||||
|
Only this declaration type may refer to an earlier run; other declared objects
|
||||||
|
remain confined to the selected commit's run.
|
||||||
|
|
||||||
The reader opens each small control object directly and enforces owner-specific
|
The reader opens each small control object directly and enforces owner-specific
|
||||||
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||||
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
|
immutable commit manifest, and 8 MiB for the selected session manifest. These
|
||||||
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
|
are exposed as `MaxCurrentCommitPointerBytes`,
|
||||||
manifest limit to `current/manifest.json`. These are exposed as
|
`MaxRemoteCommitManifestBytes`, and `MaxRemoteSessionManifestBytes`.
|
||||||
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
|
|
||||||
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
|
|
||||||
`MaxLegacyCurrentManifestBytes`.
|
|
||||||
|
|
||||||
Each read uses the generation and size metadata returned with its opened body.
|
Each read uses the generation and size metadata returned with its opened body.
|
||||||
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||||
absent or inaccurate. Immutable selections then retain their declared-size,
|
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||||
checksum, generation, and identity checks. No current-state control object is
|
checksum, generation, and identity checks. No current-state control object is
|
||||||
downloaded through a temporary file.
|
downloaded through a temporary file. Artifact and previous-session downloads
|
||||||
|
validate their complete sibling temporary file against available committed
|
||||||
|
identity before replacing a canonical local path.
|
||||||
|
|
||||||
Core helpers:
|
Core helpers:
|
||||||
|
|
||||||
@@ -191,10 +211,9 @@ Core helpers:
|
|||||||
- `ValidateCurrentStateIdentity`
|
- `ValidateCurrentStateIdentity`
|
||||||
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||||
|
|
||||||
Typed missing-state errors:
|
Typed missing-state error:
|
||||||
|
|
||||||
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
- `CurrentCommitPointerMissingError` (`ErrCurrentCommitPointerMissing`)
|
||||||
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
|
||||||
|
|
||||||
Identity validation supports caller-provided expectations:
|
Identity validation supports caller-provided expectations:
|
||||||
|
|
||||||
@@ -208,6 +227,21 @@ Caller policy is intentionally outside artifacts helpers:
|
|||||||
- some callers downgrade missing state to status/findings;
|
- some callers downgrade missing state to status/findings;
|
||||||
- some callers skip optional behavior when state is missing.
|
- some callers skip optional behavior when state is missing.
|
||||||
|
|
||||||
|
## Remote Lock Objects
|
||||||
|
|
||||||
|
The artifacts package defines the independent remote-lock payload and identity
|
||||||
|
contract. Each canonical publish source maps to one object below the validated
|
||||||
|
session `locks/` prefix; its filename is the lowercase SHA-256 digest of the
|
||||||
|
exact source followed by `.lock`.
|
||||||
|
|
||||||
|
Lock payloads are strict versioned JSON containing the canonical source, an
|
||||||
|
optional reason, and a UTC creation time. Reads are bounded to
|
||||||
|
`MaxRemoteLockObjectBytes` (64 KiB), reject unknown fields and trailing values,
|
||||||
|
and verify that the payload source matches the key digest. Exact lookup reports
|
||||||
|
presence explicitly, prefix listing validates every object and sorts by source,
|
||||||
|
creation uses an absence precondition, forced replacement uses one observed
|
||||||
|
generation, and removal is idempotent.
|
||||||
|
|
||||||
## Key Path Helpers
|
## Key Path Helpers
|
||||||
|
|
||||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||||
@@ -220,8 +254,13 @@ Caller policy is intentionally outside artifacts helpers:
|
|||||||
New publication creates run-scoped immutable objects, including
|
New publication creates run-scoped immutable objects, including
|
||||||
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||||
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||||
commit and declared object generations/checksums. Legacy current-pair loading
|
commit and declared object generations/checksums.
|
||||||
is confined to `current_state_legacy.go` for migration only.
|
|
||||||
|
A `published_output` declaration may identify its commit's run or an earlier
|
||||||
|
immutable run in the same campaign/session namespace. The key parser validates
|
||||||
|
that namespace, rejects mutable and malformed keys, and derives the originating
|
||||||
|
run from the exact declaration. All other commit artifacts remain confined to
|
||||||
|
the commit's own run prefix.
|
||||||
|
|
||||||
Campaign, session, and Narratio run IDs are validated as portable opaque
|
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||||
segments at configuration and artifact boundaries before they can be used in a
|
segments at configuration and artifact boundaries before they can be used in a
|
||||||
@@ -252,7 +291,6 @@ physical layout.
|
|||||||
`internal/artifacts/prepared_input.go`
|
`internal/artifacts/prepared_input.go`
|
||||||
- Current state: `internal/artifacts/current_state.go`,
|
- Current state: `internal/artifacts/current_state.go`,
|
||||||
`internal/artifacts/current_state_commit.go`,
|
`internal/artifacts/current_state_commit.go`,
|
||||||
`internal/artifacts/current_state_legacy.go`
|
|
||||||
- Paths and keys: `internal/artifacts/paths.go`,
|
- Paths and keys: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/s3_keys.go`
|
`internal/artifacts/s3_keys.go`
|
||||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||||
|
|||||||
@@ -11,6 +11,11 @@ physical restore scope belong in
|
|||||||
Restore separates remote authority, local conflict policy, and filesystem
|
Restore separates remote authority, local conflict policy, and filesystem
|
||||||
mutation so each remains testable independently.
|
mutation so each remains testable independently.
|
||||||
|
|
||||||
|
The public command supplies production defaults to one private,
|
||||||
|
invocation-scoped dependency value. Discovery, planning, and execution use that
|
||||||
|
same value, so focused sequencing tests can replace one boundary without
|
||||||
|
mutable package hooks.
|
||||||
|
|
||||||
## Discovery Contract
|
## Discovery Contract
|
||||||
|
|
||||||
Discovery delegates current-state pointer and manifest loading to
|
Discovery delegates current-state pointer and manifest loading to
|
||||||
@@ -34,18 +39,25 @@ Planner behavior:
|
|||||||
|
|
||||||
- a new-protocol restore uses only the selected commit's declared artifact set;
|
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||||
each action carries that artifact's immutable key, checksum, size, and
|
each action carries that artifact's immutable key, checksum, size, and
|
||||||
generation. Coherent legacy state remains on the isolated compatibility path;
|
generation;
|
||||||
- remote-to-local mapping is traversal-safe;
|
- remote-to-local mapping is traversal-safe;
|
||||||
- actions are sorted by local relative path and then remote key;
|
- actions are sorted by local relative path and then remote key;
|
||||||
|
- planning opens the selected session root and inspects every target by its
|
||||||
|
normalized session-relative path. Missing ancestors produce a missing target;
|
||||||
|
an ancestor or leaf symlink is a non-regular conflict and is never opened for
|
||||||
|
hashing;
|
||||||
- force converts differing eligible regular files from conflicts to downloads;
|
- force converts differing eligible regular files from conflicts to downloads;
|
||||||
directories and other non-regular targets remain conflicts.
|
directories, symlinks, and other non-regular targets remain conflicts.
|
||||||
|
|
||||||
For a non-dry-run restore, planning/classification happens only after acquiring
|
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||||
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||||
|
|
||||||
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||||
prepare, status, and validation. A committed source is selected only by its
|
prepare, status, and validation. Each requirement selects the unique
|
||||||
exact source identity; legacy fallback remains isolated and rejects ambiguity.
|
`published_output` for its canonical source in the pointer-selected commit,
|
||||||
|
then strips only that run's immutable `published/` prefix for its safe local
|
||||||
|
cache path. It never probes candidate objects or derives paths from manifests,
|
||||||
|
configuration, or basenames.
|
||||||
|
|
||||||
## Execution Contract
|
## Execution Contract
|
||||||
|
|
||||||
@@ -53,10 +65,14 @@ Execution order and safety:
|
|||||||
|
|
||||||
- non-manifest downloads happen before manifest install;
|
- non-manifest downloads happen before manifest install;
|
||||||
- `manifest.json` installs last;
|
- `manifest.json` installs last;
|
||||||
- downloads use sibling temp files plus atomic rename;
|
- downloads use sibling temp files, validate the complete object, then use an
|
||||||
|
atomic rename;
|
||||||
- manifest replacement is validated before rename;
|
- manifest replacement is validated before rename;
|
||||||
- each committed object is verified against its declared checksum, size, and
|
- each committed object is verified against its declared checksum, size, and
|
||||||
generation before installation;
|
generation before installation;
|
||||||
|
- installation reopens the selected session root, creates only validated
|
||||||
|
missing directory components with workspace modes, and rejects symlinked or
|
||||||
|
non-regular ancestors and leaves before replacement;
|
||||||
- a committed manifest already verified during discovery is retained for the
|
- a committed manifest already verified during discovery is retained for the
|
||||||
matching restore action and revalidated before installation, avoiding a
|
matching restore action and revalidated before installation, avoiding a
|
||||||
second body transfer;
|
second body transfer;
|
||||||
@@ -67,15 +83,17 @@ Execution order and safety:
|
|||||||
- restored manifest local references are rebased beneath the selected local
|
- restored manifest local references are rebased beneath the selected local
|
||||||
session root. Unsafe relative references and producer-machine absolute paths
|
session root. Unsafe relative references and producer-machine absolute paths
|
||||||
outside the manifest's producer session root are rejected; producer-local
|
outside the manifest's producer session root are rejected; producer-local
|
||||||
spool/cache and cleanup locations are not restored as authority.
|
spool paths and run-local diagnostic references are not restored as
|
||||||
|
authority. Cleanup remains ordinary restored stage state and is not reset or
|
||||||
|
synthesized by restore.
|
||||||
|
|
||||||
Audio restore path:
|
Audio restore path:
|
||||||
|
|
||||||
- uses `audio.MaterializeS3Audio`;
|
- uses `audio.MaterializeS3Audio`;
|
||||||
- integrates spool and S3 audio cache paths;
|
- integrates spool and S3 audio cache paths;
|
||||||
- reuses cached audio only when its no-follow regular file, content digest, and
|
- reuses cached audio only when its opened regular file, content digest, and
|
||||||
identity sidecar all match the selected remote object version; otherwise it
|
identity sidecar all match the selected remote object version; otherwise it
|
||||||
refreshes through the durable download path.
|
refreshes through the validated download path.
|
||||||
|
|
||||||
## Reporting Contract
|
## Reporting Contract
|
||||||
|
|
||||||
@@ -92,9 +110,7 @@ Audio restore path:
|
|||||||
loaded at discovery; later pointer changes cannot add objects or substitute a
|
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||||
different run into its plan;
|
different run into its plan;
|
||||||
- a verified `current/commit-pointer.json` and its selected immutable commit
|
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||||
establish new-protocol remote commitment; coherent legacy
|
establish remote commitment; when absent, republish the session;
|
||||||
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
|
|
||||||
support;
|
|
||||||
- restore does not execute pipeline stages.
|
- restore does not execute pipeline stages.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
@@ -107,4 +123,5 @@ Audio restore path:
|
|||||||
- Tests: `internal/app/restore_discovery_test.go`,
|
- Tests: `internal/app/restore_discovery_test.go`,
|
||||||
`internal/app/restore_plan_test.go`,
|
`internal/app/restore_plan_test.go`,
|
||||||
`internal/app/restore_execution_test.go`,
|
`internal/app/restore_execution_test.go`,
|
||||||
|
`internal/app/restore_recovery_test.go`,
|
||||||
`internal/app/restore_workflow_test.go`
|
`internal/app/restore_workflow_test.go`
|
||||||
|
|||||||
@@ -8,29 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
|
|||||||
|
|
||||||
`LoadPipeline` assembles and validates a pipeline in this order:
|
`LoadPipeline` assembles and validates a pipeline in this order:
|
||||||
|
|
||||||
1. Parse the root YAML into a presence-aware composition tree. The tree retains
|
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
|
||||||
source names, full field paths, node kinds, declaration order, and explicit
|
enabled imports, and the caller's profile selection. Omitted, empty, and
|
||||||
zero, false, empty-map, and empty-list values.
|
exact `default` selections use the base; named profiles use the fixed
|
||||||
2. Remove the root-only `composition` envelope and validate its explicit
|
`profiles/<name>.yml` convention.
|
||||||
`imports`, `default_profile`, and named `profiles` declarations. A load
|
2. ConfigKit parses the root and declared imports, additively merges base
|
||||||
option retains the difference between omitted and explicitly empty profile
|
sources, applies only the selected overlay, and strictly decodes the result.
|
||||||
selection.
|
3. The definition resolver applies recorded entry-level and inherited defaults,
|
||||||
3. Open each import relative to the root pipeline directory through the
|
runs deterministic domain normalization, records derived publish destinations,
|
||||||
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
|
and resolves ordinary paths relative to the root pipeline file.
|
||||||
extension and cannot traverse, use symlinks, repeat a file, import the root,
|
4. The definition validator performs read-only checks that require only the
|
||||||
or contain another composition envelope.
|
pipeline file. Checks requiring canonical-party family expansion remain at
|
||||||
4. Resolve and structurally parse every declared profile overlay through the
|
the existing explicit pipeline/campaign resolution boundary.
|
||||||
same confined regular-file boundary. Missing or malformed unselected
|
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
|
||||||
overlays fail the load. Overlays cannot contain a composition envelope.
|
declared by multiple base files, adapts source ownership records, attaches
|
||||||
5. Additively merge the root body and imports. Distinct map leaves compose;
|
runtime-only state, and computes the normalized effective digest.
|
||||||
repeated scalar or list paths and node-kind disagreements are conflicts.
|
|
||||||
6. Select exactly one declared profile from an explicit option or the default,
|
|
||||||
then recursively merge its overlay. Overlay leaves replace base leaves,
|
|
||||||
lists are atomic replacements, and null or kind changes fail.
|
|
||||||
7. Emit deterministic canonical YAML and strictly decode it into
|
|
||||||
`PipelineConfig`.
|
|
||||||
8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from
|
|
||||||
the root pipeline file, and digest the normalized effective mapping.
|
|
||||||
|
|
||||||
This ordering preserves monolithic configuration behavior. Moving a field to
|
This ordering preserves monolithic configuration behavior. Moving a field to
|
||||||
an imported fragment changes its source ownership, not its path base, default,
|
an imported fragment changes its source ownership, not its path base, default,
|
||||||
@@ -52,109 +44,152 @@ download retain that exact pipeline object and its private provenance. Removing
|
|||||||
a temporary downloaded session file therefore cannot invalidate the resolved
|
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||||
pipeline or campaign context.
|
pipeline or campaign context.
|
||||||
|
|
||||||
|
Campaign and session files use independent ConfigKit definitions with imports
|
||||||
|
and profiles disabled. Campaign files normally load directly from their source;
|
||||||
|
session files are first read through the 8 MiB bounded-read boundary because the
|
||||||
|
application must reject unresolved template placeholders before passing an exact
|
||||||
|
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
|
||||||
|
the same typed loader and retain the existing post-load identity expectations,
|
||||||
|
stable-input precedence, and source-relative path semantics.
|
||||||
|
|
||||||
The application also has a separate read-only inspection resolver for `config
|
The application also has a separate read-only inspection resolver for `config
|
||||||
validate`, `config show`, and `config sources`. It uses the same production root/profile and
|
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
|
||||||
campaign selection functions, but never routes through session discovery,
|
campaign selection functions, but never routes through session discovery,
|
||||||
remote-session download, secret loading, adapter composition, workspace
|
remote-session download, adapter composition, workspace
|
||||||
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
||||||
family declarations must resolve its selected campaign before ordinary pipeline
|
family declarations must resolve its selected campaign before ordinary pipeline
|
||||||
validation, which expands its canonical-party members and generated publish
|
validation, which expands its canonical-party members and generated publish
|
||||||
rules. A pipeline without those declarations may be validated by itself.
|
rules. A pipeline without those declarations may be validated by itself.
|
||||||
|
Secret-directory bootstrap remains invocation-scoped and happens before the
|
||||||
|
ordinary ConfigKit load because explicit typed references must resolve even for
|
||||||
|
inspection.
|
||||||
|
|
||||||
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
||||||
show`. It serializes the typed, defaulted effective mapping through the
|
show`. It serializes one explicit typed safe projection with the existing YAML
|
||||||
deterministic composition renderer, then removes resolution-only artifact
|
encoder. The projection has no slots for resolution-only artifact family
|
||||||
family declarations. The result contains no composition envelope or private
|
declarations or private provenance. The result has one trailing newline;
|
||||||
provenance fields and has one trailing newline; commands do not marshal runtime
|
commands do not marshal runtime objects directly.
|
||||||
objects directly.
|
|
||||||
|
|
||||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||||
the complete logical field paths retained during composition and classifies
|
the complete logical field paths projected from the schema and classifies
|
||||||
each contributor as root, import, profile, or centralized default. The
|
each contributor as root, import, profile, or centralized default. The
|
||||||
projection replaces generated concrete member paths with paired family and
|
projection replaces generated concrete member paths with paired family and
|
||||||
canonical-party records, and does the same for generated publish rules.
|
canonical-party records, and does the same for generated publish rules.
|
||||||
Campaign records identify campaign-owned fields and party inputs; canonical
|
Campaign records identify campaign-owned fields and party inputs; canonical
|
||||||
derived players point to the party source, while legacy players retain a
|
derived players point to the party source. The application command only joins these sorted
|
||||||
dedicated legacy-player role. The application command only joins these sorted
|
|
||||||
records with selection metadata and never reparses configuration files.
|
records with selection metadata and never reparses configuration files.
|
||||||
|
|
||||||
`config diff` uses a paired profile loader that parses the root, imports, and
|
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||||
declared overlays once, then clones the additive base before independently
|
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||||
selecting, decoding, defaulting, and finalizing each profile. When campaign
|
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||||
|
family/character/party provenance for generated members. It does not inspect
|
||||||
|
files named after artifacts or infer another runtime graph.
|
||||||
|
|
||||||
|
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||||
|
loads. When campaign
|
||||||
resolution is needed, the command loads one selected campaign and party and
|
resolution is needed, the command loads one selected campaign and party and
|
||||||
expands both effective pipelines from that same party value. The configuration
|
expands both effective pipelines from that same party value. The configuration
|
||||||
owner projects each normalized effective mapping into sorted logical paths;
|
owner projects each normalized effective mapping directly from that same typed
|
||||||
|
projection into sorted logical paths using JSON number preservation;
|
||||||
mapping leaves are compared individually while sequence values remain atomic.
|
mapping leaves are compared individually while sequence values remain atomic.
|
||||||
Values are compact deterministic JSON representations for command output, not
|
Values are compact deterministic JSON representations for command output, not
|
||||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||||
with no projected difference is treated as an internal consistency error.
|
with no projected difference is treated as an internal consistency error.
|
||||||
|
|
||||||
Campaign context construction also reads and classifies the campaign-owned
|
Campaign context construction reads the campaign-owned party source through
|
||||||
party source through `ParseParty`. A canonical party retains its raw bytes and
|
`ParseParty`. The required canonical party retains its raw bytes and normalized
|
||||||
normalized roster in runtime-only `ResolvedParty` provenance, while a legacy
|
roster in runtime-only `ResolvedParty` provenance. Canonical resolution creates
|
||||||
party remains opaque. Canonical resolution creates a virtual
|
a deterministic `derived_from_party` players projection. The party is the only
|
||||||
`derived_from_party` players input and rejects competing campaign or session
|
configured roster authority; player and session-party fields are unsupported.
|
||||||
players files and session party overrides. The compact legacy compatibility
|
|
||||||
path resolves the effective campaign/session party and players files together.
|
|
||||||
|
|
||||||
## Canonical Party Domain
|
## Canonical Party Domain
|
||||||
|
|
||||||
`ParseParty` is the package-owned boundary for classifying a party source.
|
`ParseParty` is the package-owned boundary for parsing a party source. Its
|
||||||
When a top-level `schema_version` is present, it strictly validates the
|
single-source ConfigKit definition disables imports and profiles, requires
|
||||||
`narratio.party.v1` contract into ordered character domain values. The
|
`schema_version` through `VersionRule`, and strictly decodes the
|
||||||
canonical value retains a separate exact byte copy of its source so consumers
|
character-keyed typed schema before applying Narratio's party-domain checks.
|
||||||
can materialize the authored party document without reserializing it. Its
|
Successful metadata coordinates recover authored character-map order without a
|
||||||
`PlayersYAML` method deterministically derives the versioned players-only
|
second YAML parse; class and alias slice order comes directly from typed decode.
|
||||||
projection.
|
The canonical value retains a separate exact byte copy of its source so
|
||||||
|
consumers can materialize the authored party document without reserializing it.
|
||||||
|
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
|
||||||
|
players-only projection.
|
||||||
|
|
||||||
An unversioned source is classified by the small legacy compatibility boundary
|
An unversioned source is rejected before resolution, family expansion, or
|
||||||
in `party_legacy.go`; it deliberately exposes no parsed roster information.
|
prepare-stage materialization.
|
||||||
That boundary exists solely to isolate removable compatibility behavior from
|
|
||||||
the canonical parser.
|
## Validation Ownership
|
||||||
|
|
||||||
|
Configuration validation remains within `internal/config`, with the public
|
||||||
|
entry points and pipeline-domain sequencing separated from the private owners
|
||||||
|
for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
||||||
|
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||||
|
boundaries rather than extension points or a general validation framework.
|
||||||
|
|
||||||
|
Validation is read-only, fails fast, and does not define which of several
|
||||||
|
independent invalid fields is reported first. Map-backed declarations are
|
||||||
|
nevertheless traversed deterministically. Reusable domain normalization is an
|
||||||
|
explicit preceding operation for both loaded and programmatically constructed
|
||||||
|
pipelines. Notarius normalization commits canonical scalar, reference, and output
|
||||||
|
values only after the complete normalization succeeds; its validator then checks
|
||||||
|
that candidate without changing it. Scriptorium validates artifact declarations
|
||||||
|
and outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||||
|
dependency graph in deterministic artifact order.
|
||||||
|
|
||||||
|
Diagnostics identify the relevant semantic field. Merge conflicts also
|
||||||
|
retain every claiming source needed to correct an additive ownership conflict.
|
||||||
|
Complete prose, punctuation, formatting, and private validation choreography are
|
||||||
|
not compatibility contracts.
|
||||||
|
|
||||||
## Diagnostics And Runtime Metadata
|
## Diagnostics And Runtime Metadata
|
||||||
|
|
||||||
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
Syntax, duplicate-key, merge, profile, and schema failures include the
|
||||||
relevant source name and full field path. Additive conflicts report every
|
relevant source name and full field path. Additive conflicts report every
|
||||||
claiming source so operators can repair the split without repeatedly
|
claiming source so operators can repair the split without repeatedly
|
||||||
rediscovering additional conflicts.
|
rediscovering additional conflicts.
|
||||||
|
|
||||||
The loaded pipeline retains private runtime metadata for the absolute root
|
The loaded pipeline retains private runtime metadata for the absolute root
|
||||||
path, ordered imports, selected profile name and selection source, selected
|
path, ordered imports, selected profile name and selection source, selected
|
||||||
overlay, contributing sources, effective digest, and leaf ownership. Base
|
overlay, original ConfigKit metadata and source snapshots, effective digest,
|
||||||
|
and leaf ownership. Base
|
||||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
leaves retain their root/import owners, replaced leaves belong to the selected
|
||||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
overlay, and centrally supplied values use the synthetic `default` owner. This
|
||||||
metadata does not participate in YAML decoding or alter the public
|
metadata does not participate in YAML decoding or alter the public
|
||||||
configuration model.
|
configuration model.
|
||||||
|
|
||||||
The effective digest is SHA-256 over deterministic canonical YAML produced from
|
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
|
||||||
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
|
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
|
||||||
execution, but the digest substitutes their normalized logical values captured
|
execution, but the digest substitutes their normalized logical values captured
|
||||||
before root-relative resolution, so relocating an equivalent configuration
|
before root-relative resolution, so relocating an equivalent configuration
|
||||||
bundle does not change provenance. Because composition and resolution metadata
|
bundle does not change provenance. Because ConfigKit and resolution metadata
|
||||||
are private, the digest excludes source layout, profile name, and ownership.
|
are private, the digest excludes source layout, profile name, and ownership.
|
||||||
Configuration stores environment variable names rather than resolving raw
|
Configuration stores credentials as ConfigKit typed references. A successful
|
||||||
credentials, so raw secret values are neither loaded nor hashed.
|
load resolves their values inside the opaque typed boundary, while this safe
|
||||||
|
projection serializes only `EnvName()` identities. Credential-value changes are
|
||||||
|
therefore never hashed; reference-name changes remain ordinary provenance.
|
||||||
|
The optional Notarius credential follows the same path even for disabled or
|
||||||
|
resumable extraction; normalization copies the opaque reference without reading
|
||||||
|
or resolving it again. Direct YAML serialization has a configuration-owned safe
|
||||||
|
projection so only the reference name can leave this boundary.
|
||||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||||
for later runtime expansion.
|
for later runtime expansion.
|
||||||
|
|
||||||
## Test Surfaces
|
## Test Surfaces
|
||||||
|
|
||||||
`composition_test.go` protects the presence and merge algebra independently of
|
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
|
||||||
the public schema. `pipeline_composition_test.go` exercises explicit imports,
|
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
||||||
confinement, conflicts, strict decoding, metadata, and root-relative path
|
unique artifact/family base-declaration rule. Existing focused configuration
|
||||||
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||||
all-overlay validation, overlay behavior, provenance, option propagation, and
|
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||||
effective-digest stability. Application configuration-loader tests protect the
|
|
||||||
single-read boundary by changing the pipeline file after its initial load and
|
single-read boundary by changing the pipeline file after its initial load and
|
||||||
confirming local session resolution retains the original pipeline. Other
|
confirming local session resolution retains the original pipeline. Other
|
||||||
configuration tests continue to protect defaults and validation after assembly.
|
configuration tests continue to protect defaults and validation after assembly.
|
||||||
`party_test.go` protects the versioned party schema, domain invariants, and
|
`party_test.go` protects the versioned typed party schema, domain invariants,
|
||||||
deterministic players projection without involving campaign or runtime wiring.
|
authored character order, retained bytes, and deterministic players projection
|
||||||
|
without involving campaign or runtime wiring.
|
||||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||||
input restrictions, legacy overrides, source provenance, and virtual players
|
input restrictions, source provenance, and virtual players input selection.
|
||||||
input selection.
|
|
||||||
|
|
||||||
## Artifact Family Resolution
|
## Artifact Family Resolution
|
||||||
|
|
||||||
|
|||||||
@@ -1,71 +1,55 @@
|
|||||||
# Internal: File Operations
|
# Internal: File Operations
|
||||||
|
|
||||||
`internal/fileops` owns the narrow mechanics for durable replacement of one
|
`internal/fileops` owns the narrow mechanics for atomically visible replacement of one
|
||||||
byte file. Callers keep ownership of serialization, validation, cancellation,
|
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||||
and destination-directory policy.
|
and destination-directory policy.
|
||||||
|
|
||||||
## Destination Confinement
|
## Destination Paths
|
||||||
|
|
||||||
Before it creates, replaces, or installs a destination file, `fileops` opens
|
Ordinary workspace replacement uses standard operating-system path semantics.
|
||||||
each ancestor from the filesystem root and rejects symbolic links or components
|
Callers with a root-scoped safety contract own that validation at their boundary.
|
||||||
that change during traversal. The resulting parent-directory handle is retained
|
|
||||||
for sibling temporary-file creation and rename, so a later pathname swap cannot
|
|
||||||
redirect the replacement. Existing destination symlinks are replaced as leaf
|
|
||||||
entries; their targets are never followed.
|
|
||||||
|
|
||||||
Remote object acquisition uses a writer supplied by the storage owner. The
|
Remote object acquisition is owned by its domain caller. The caller writes a
|
||||||
writer receives a `fileops`-owned, already-open sibling temporary file rather
|
sibling temporary file, validates the complete result, and passes that file to
|
||||||
than a mutable destination path. Callers still own remote object selection,
|
`InstallSiblingTempFile` only when it is ready to become canonical. Callers own
|
||||||
validation, conflict handling, and final mode.
|
remote object selection, validation, conflict handling, and temporary-file
|
||||||
|
cleanup.
|
||||||
|
|
||||||
Directory promotion keeps the verified destination parent open while it creates
|
Directory bundles are not promoted by `internal/fileops`; extraction owns its
|
||||||
the temporary tree, copies regular source entries, and performs the platform
|
validated Notarius bundle rename.
|
||||||
no-replace rename. Platforms without a verified handle-relative atomic
|
|
||||||
no-replace primitive reject promotion before writing a temporary tree.
|
|
||||||
|
|
||||||
## Cleanup Contract
|
## Cleanup Contract
|
||||||
|
|
||||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
|
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant, opens
|
||||||
the root and each target ancestor without following symlinks, then removes the
|
that root with `os.OpenRoot`, and removes the normalized root-relative target
|
||||||
tree through those directory handles. It rejects root deletion and any symlink
|
with `Root.RemoveAll`. It rejects root, traversal, and out-of-root targets; a
|
||||||
encountered in the target path or tree; repeated removal of a missing target is
|
missing target is successful. `RemoveFileUnderRoot` additionally requires an
|
||||||
successful. Command and post-publish policy remains owned by `internal/app`.
|
existing leaf to be regular. Command and post-publish policy remains owned by
|
||||||
|
`internal/app`.
|
||||||
|
|
||||||
## Confined Reads
|
## Bounded Reads
|
||||||
|
|
||||||
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
|
`ReadFileBounded` opens a normal path and reads at most its caller-supplied
|
||||||
caller-selected root and relative file path; `ReadRegularFile` is its
|
limit. Credential mode policy and environment precedence remain owned by
|
||||||
path-based convenience wrapper. They verify every ancestor through directory
|
`internal/app`.
|
||||||
handles and admit only a stable regular-file handle. Callers enforce their own
|
|
||||||
byte limits and access policy. Credential mode policy and environment
|
|
||||||
precedence remain owned by `internal/app`.
|
|
||||||
|
|
||||||
## Replacement Contract
|
## Replacement Contract
|
||||||
|
|
||||||
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||||
sibling temporary file, writes the complete byte sequence, applies the
|
sibling temporary file, writes the complete byte sequence, applies the
|
||||||
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
|
caller-supplied mode, closes the file, runs an optional pre-rename check, and
|
||||||
check, replaces the destination with a rename, then syncs the containing
|
replaces the destination with a rename.
|
||||||
directory.
|
|
||||||
|
|
||||||
The pre-rename check is the last point at which a caller can cancel without
|
The pre-rename check is the last point at which a caller can cancel without
|
||||||
installing a new destination. A failure before the rename leaves the old
|
installing a new destination. A failure before the rename leaves the old
|
||||||
destination unchanged and removes the temporary file; any cleanup failure is
|
destination unchanged and removes the temporary file. Successful replacement
|
||||||
returned alongside the primary failure. A failure after the rename may leave
|
provides atomic visibility; it makes no crash- or power-loss durability claim.
|
||||||
the new file visible, but it is not reported as crash-durable.
|
|
||||||
|
|
||||||
Replacement follows the operating system's same-filesystem rename semantics.
|
Replacement follows the operating system's same-filesystem rename semantics.
|
||||||
If a platform cannot replace an existing destination, the operation returns an
|
If a platform cannot replace an existing destination, the operation returns an
|
||||||
error and never removes the old file as an emulation step.
|
error and never removes the old file as an emulation step.
|
||||||
|
|
||||||
## Directory-Sync Support
|
`WriteFileAtomic` and the copy helpers create the destination parent with the
|
||||||
|
repository's workspace permissions. `InstallSiblingTempFile` requires the
|
||||||
Linux and macOS attempt to sync the destination directory. Windows opens the
|
caller to provide an existing temporary file in the existing destination
|
||||||
directory with backup semantics and flushes its buffers. If either operation
|
directory.
|
||||||
is unavailable for the platform, directory handle, or filesystem,
|
|
||||||
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
|
|
||||||
as successful crash-durable replacement.
|
|
||||||
|
|
||||||
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
|
|
||||||
retain their compatibility behavior of creating the destination parent with
|
|
||||||
the repository's workspace permissions before using this contract.
|
|
||||||
|
|||||||
@@ -2,29 +2,38 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Explain the session-progress and invocation-audit models implemented by
|
Explain the session-progress model implemented by `internal/manifest`. Physical
|
||||||
`internal/manifest`. Physical manifest placement belongs in
|
manifest placement belongs in
|
||||||
[Operations](../operations.md#local-state-layout).
|
[Operations](../operations.md#local-state-layout).
|
||||||
|
|
||||||
## Session Manifest
|
## Session Manifest
|
||||||
|
|
||||||
`manifest.Manifest` records:
|
`manifest.Manifest` records:
|
||||||
|
|
||||||
|
- required `schema_version: narratio.session-manifest.v1`
|
||||||
- identity (`session_id`, `campaign`, `run_id`)
|
- identity (`session_id`, `campaign`, `run_id`)
|
||||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||||
- `inputs` records
|
- `inputs` records
|
||||||
- durable `artifacts` records
|
- durable `artifacts` records
|
||||||
- per-stage `stages` map
|
- per-stage `stages` map, including the ordinary cleanup stage record
|
||||||
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
|
||||||
remote commit identity, and each exact root-confined local target to its
|
|
||||||
completion evidence
|
|
||||||
|
|
||||||
Session, campaign, and run identities in local and downloaded manifests must be
|
Session, campaign, and run identities in local and downloaded manifests must be
|
||||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||||
guidance rather than being normalized into a different workspace or remote
|
guidance rather than being normalized into a different workspace or remote
|
||||||
namespace.
|
namespace.
|
||||||
|
|
||||||
|
Decoding is strict at every typed manifest field. Unknown fields are rejected;
|
||||||
|
the schema version remains `narratio.session-manifest.v1` and there is no
|
||||||
|
compatibility reader that silently drops retired state. Cleanup progress uses
|
||||||
|
only the cleanup entry in `stages`.
|
||||||
|
|
||||||
|
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
||||||
|
non-nil and its embedded `name` exactly matches its map key; loading never
|
||||||
|
invents either marker. A missing or unsupported schema version, or incomplete
|
||||||
|
stage identity, is unsupported local state: move it aside and regenerate it,
|
||||||
|
or restore a verified current committed snapshot when that is authoritative.
|
||||||
|
|
||||||
Prepare records independent `party` and `players` input checksums. In canonical
|
Prepare records independent `party` and `players` input checksums. In canonical
|
||||||
party mode, the party record retains its campaign source identity while the
|
party mode, the party record retains its campaign source identity while the
|
||||||
players record uses `derived_from_party`; raw roster content is never embedded
|
players record uses `derived_from_party`; raw roster content is never embedded
|
||||||
@@ -41,13 +50,52 @@ The model admits these stage states:
|
|||||||
- `stale`
|
- `stale`
|
||||||
- `interrupted`
|
- `interrupted`
|
||||||
|
|
||||||
|
### Transcribe-owned speaker state
|
||||||
|
|
||||||
|
A `transcribe` stage record may carry `transcribe_state_version: 1` and a
|
||||||
|
`transcribe_speakers` map keyed by the existing derived speaker ID. The version
|
||||||
|
is the authority marker: a versioned empty map means that no earlier speaker
|
||||||
|
completion is reusable. Absence of both fields is accepted as legacy
|
||||||
|
aggregate-only state so current Narratio can read older manifests.
|
||||||
|
|
||||||
|
Each map entry represents one completed speaker only. It binds a versioned
|
||||||
|
fingerprint to the session-relative audio path, audio SHA-256 and size, one
|
||||||
|
canonical `transcript_raw` output with its path, SHA-256 and size, and the
|
||||||
|
producer run ID. Paths must be canonical and session-relative, checksums must
|
||||||
|
be lowercase SHA-256 values, and the record and output producer identities
|
||||||
|
must agree. Other stages cannot carry this transcribe-owned state.
|
||||||
|
Audio size is nonnegative; WhisperX determines whether that audio can be
|
||||||
|
transcribed. Output size remains positive and the transcript must be valid JSON.
|
||||||
|
|
||||||
|
Transcribe reconciles this collection against the current prepared audio set
|
||||||
|
and observable WhisperX semantics. Missing or corrupt regular outputs become
|
||||||
|
work; unsafe or non-regular evidence is rejected. It saves the reconciled
|
||||||
|
collection before HTTP dispatch and saves a detached candidate after each
|
||||||
|
validated completion. Only after a save succeeds is the candidate installed as
|
||||||
|
live authority. Canonical bytes produced before an unsuccessful save are not a
|
||||||
|
completion record.
|
||||||
|
|
||||||
|
Aggregate transcribe status still describes the whole invocation. A failed,
|
||||||
|
running, stale, or interrupted aggregate may therefore retain reusable speaker
|
||||||
|
records, while aggregate outputs are declared only after the current
|
||||||
|
collection is complete. Ordinary retry reconciles and reuses those records.
|
||||||
|
Force first checkpoints an empty current collection, preventing later retries
|
||||||
|
from reviving pre-force results.
|
||||||
|
|
||||||
|
Older aggregate-only transcribe success is conservatively regenerated once
|
||||||
|
when transcribe is selected because it does not bind individual outputs to the
|
||||||
|
audio bytes used. The session-manifest schema identifier remains
|
||||||
|
`narratio.session-manifest.v1`; this additive state is accepted by current
|
||||||
|
Narratio, but older strict-decoding binaries may reject manifests after these
|
||||||
|
fields have been written.
|
||||||
|
|
||||||
### Analyze-owned artifact state
|
### Analyze-owned artifact state
|
||||||
|
|
||||||
The `analyze` stage record may carry `analyze_state_version: 1` and an
|
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
||||||
`analyze_artifacts` map keyed by normalized configured artifact key. The
|
`analyze_artifacts` map keyed by normalized configured artifact key. The
|
||||||
version is the authority marker: version 1 with no entries is a valid evaluated
|
version is the authority marker: version 1 with no entries is a valid evaluated
|
||||||
empty set, while an absent version is legacy aggregate-only state and provides
|
empty set. An absent or unsupported version is unsupported local state and is
|
||||||
no current configured-artifact evidence.
|
rejected rather than being treated as aggregate-only authority.
|
||||||
|
|
||||||
Each analyze artifact record has one disposition:
|
Each analyze artifact record has one disposition:
|
||||||
|
|
||||||
@@ -72,18 +120,15 @@ do not replace the concrete configured key or infer family membership from a
|
|||||||
name, so older records without them remain valid.
|
name, so older records without them remain valid.
|
||||||
|
|
||||||
The session-stage collection is the reconciled authority across invocations.
|
The session-stage collection is the reconciled authority across invocations.
|
||||||
The corresponding collection on an invocation's `analyze` stage record is an
|
These records remain analyze-owned data inside the fixed stage; they are not
|
||||||
audit of only the artifacts evaluated or attempted by that run. These records
|
dynamic stages or generic subtasks.
|
||||||
remain analyze-owned data inside the fixed stage; they are not dynamic stages
|
|
||||||
or generic subtasks.
|
|
||||||
|
|
||||||
The stage result contract has one analyze-specific projection boundary. On
|
Analyze checkpoints one complete validated candidate collection after each
|
||||||
success, the runner validates and deep-copies the complete reconciled session
|
successful artifact and before exposing that artifact to later work. It saves a
|
||||||
collection and the invocation subset. Aggregate session outputs are rebuilt in
|
detached manifest candidate and installs its version, collection, and updated
|
||||||
configured-key order from current session records only; invocation outputs are
|
timestamp into live state only after the store reports success. Aggregate stage
|
||||||
limited to current records produced by that invocation's run ID. Ordinary
|
outputs are ordinary result outputs rebuilt in configured-key order from current
|
||||||
stage outputs cannot accompany this projection, so there is one source of
|
session records only.
|
||||||
artifact authority.
|
|
||||||
|
|
||||||
Successful incremental execution replaces only evaluated artifact records and
|
Successful incremental execution replaces only evaluated artifact records and
|
||||||
preserves valid unrelated current records. Rebuilt outputs are compared by
|
preserves valid unrelated current records. Rebuilt outputs are compared by
|
||||||
@@ -92,43 +137,31 @@ the same recomputed fingerprint to remain current, while a changed identity
|
|||||||
removes output authority from every unselected transitive dependent by marking
|
removes output authority from every unselected transitive dependent by marking
|
||||||
it stale. A partial analyze invocation can therefore succeed while unrelated
|
it stale. A partial analyze invocation can therefore succeed while unrelated
|
||||||
configured records remain stale. Existing canonical files never create current
|
configured records remain stale. Existing canonical files never create current
|
||||||
records without validated execution and projection.
|
records without validated execution and a successful checkpoint.
|
||||||
|
|
||||||
Aggregate analyze status is deliberately coarser than this collection. Resume
|
Aggregate analyze status is deliberately coarser than this collection. Resume
|
||||||
validation may skip a succeeded aggregate record when the selected artifact
|
validation may skip a succeeded aggregate record when the selected artifact
|
||||||
closure is current even if unrelated records are stale. Conversely, a stale
|
closure is current even if unrelated records are stale. Conversely, a stale
|
||||||
aggregate record may cross the ordinary runner boundary and perform zero
|
aggregate record may cross the ordinary runner boundary and perform zero
|
||||||
Scriptorium calls when reconciliation proves every selected artifact current;
|
Scriptorium calls when reconciliation proves every selected artifact current;
|
||||||
the successful projection then restores the aggregate status.
|
ordinary successful stage completion then restores the aggregate status.
|
||||||
|
|
||||||
Analyze may return a projection together with an error. That restricted result
|
On an artifact error, analyze installs a complete retry collection on the live
|
||||||
cannot carry ordinary outputs, skip state, aggregate logs, generated configs,
|
manifest before returning. The active record is failed without output, current
|
||||||
or metadata. The runner persists only the validated per-artifact collections,
|
transitive dependents are stale, unrelated current records survive, and only
|
||||||
then marks the aggregate analyze and run state failed and invalidates delivery
|
earlier checkpointed completions remain current. The runner's ordinary failure
|
||||||
dependents conservatively. Unrelated current records survive because the
|
path marks aggregate analyze failed, clears aggregate outputs, invalidates
|
||||||
session projection is complete. A malformed projection is not applied, and a
|
delivery dependents, and saves that collection. A terminal session-save failure
|
||||||
failed session projection save restores the prior per-artifact authority before
|
is joined with the original adapter, validation, filesystem, cancellation, or
|
||||||
terminal failure persistence.
|
checkpoint cause.
|
||||||
|
|
||||||
The incremental executor constructs this restricted projection at each
|
If an artifact checkpoint fails, analyze rolls its working collection back to
|
||||||
scheduled artifact boundary. The active record is failed without output,
|
the last committed collection before constructing the active failed record.
|
||||||
current transitive dependents are stale, unrelated current records survive, and
|
Newly materialized canonical bytes therefore remain incidental and cannot
|
||||||
only earlier validated and materialized completions remain current in the
|
become current authority. A later ordinary run reconciles the durable records,
|
||||||
invocation subset. Session failure state is persisted before invocation failure
|
reuses earlier current artifacts, and resumes with the failed or stale work.
|
||||||
state. If either terminal save fails, its persistence error is joined with the
|
The same recovery applies after process interruption: the aggregate record may
|
||||||
original adapter, validation, or filesystem cause; a failed projection save
|
be non-successful, but every completed artifact checkpoint remains reusable.
|
||||||
does not turn incidental canonical bytes into manifest authority.
|
|
||||||
|
|
||||||
## Run Manifest
|
|
||||||
|
|
||||||
`manifest.RunManifest` is created for each invocation and records:
|
|
||||||
|
|
||||||
- invocation identity and `force` flag
|
|
||||||
- the selected profile (when any) and secret-free effective configuration digest
|
|
||||||
- requested stages
|
|
||||||
- per-stage action (`run` or `skip`)
|
|
||||||
- per-stage status
|
|
||||||
- overall run status (`running`, `succeeded`, `failed`)
|
|
||||||
|
|
||||||
## Remote Commit Manifest
|
## Remote Commit Manifest
|
||||||
|
|
||||||
@@ -145,38 +178,51 @@ binds that object by checksum, size, and generation. Readers strictly reject
|
|||||||
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||||
objects that do not match their declaration.
|
objects that do not match their declaration.
|
||||||
|
|
||||||
The reader retains a temporary, clearly isolated compatibility path for a
|
An absent commit pointer means no remote state is published. Republish to
|
||||||
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
|
establish a current immutable snapshot.
|
||||||
path is removable after migration and is never used to write new state.
|
|
||||||
|
|
||||||
## Persistence Semantics
|
## Persistence Semantics
|
||||||
|
|
||||||
`manifest.LocalStore`:
|
`manifest.LocalStore`:
|
||||||
|
|
||||||
- validates loaded documents;
|
- accepts only the exact supported session-manifest schema version;
|
||||||
- normalizes missing maps/stage records;
|
- rejects unknown fields and multiple JSON values;
|
||||||
- writes through a sibling temporary file, syncing the completed file and
|
- validates stage-map identity without synthesizing missing maps, records, or
|
||||||
destination directory after atomic replacement;
|
names;
|
||||||
|
- retains only representation-preserving collection normalization;
|
||||||
|
- writes through a sibling temporary file and atomic replacement visibility;
|
||||||
- updates `updated_at` on save.
|
- updates `updated_at` on save.
|
||||||
|
|
||||||
If the operating system or filesystem cannot sync a directory, save returns an
|
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
||||||
explicit error instead of claiming crash-durable replacement. A returned error
|
durability guarantee. Callers reload after a returned save error before retrying.
|
||||||
after the rename can therefore leave the new manifest visible but not confirmed
|
A transcribe or analyze checkpoint error leaves the prior manifest collection
|
||||||
durable; callers must reload it before retrying.
|
as authority. The runner then attempts to persist the aggregate failure; if
|
||||||
|
that terminal save also fails, its error is joined with the originating failure
|
||||||
|
and the next invocation must reconcile from the last readable manifest.
|
||||||
|
|
||||||
## Execution Semantics
|
## Execution Semantics
|
||||||
|
|
||||||
The application runner marks an executing stage running and then succeeded or
|
Planning and execution share one pure pre-execution decision over the recorded
|
||||||
failed in both manifests, persisting each transition. On success it records
|
stage status, force selection, and normalized resume-validation result. That
|
||||||
outputs, logs, generated configuration references, metadata, and—when the
|
decision identifies run versus reuse and any required stale and dependent
|
||||||
stage implements the optional contract—a versioned semantic-configuration
|
invalidation transition. Planning applies it only to a cloned manifest.
|
||||||
fingerprint. Artifact
|
Execution remains the sole owner of observed stage results and persistence.
|
||||||
|
|
||||||
|
For each selected stage, the application runner computes semantic evidence,
|
||||||
|
performs any eligible resume validation, applies the pre-execution decision,
|
||||||
|
and persists required invalidation before either reusing the existing success
|
||||||
|
or beginning an attempt. The same one-stage execution owner marks the attempt
|
||||||
|
running before invoking the stage and then records and persists its failed,
|
||||||
|
skipped, or successful terminal state before a later stage may start. On
|
||||||
|
success it records outputs, logs, generated configuration references, metadata,
|
||||||
|
and—when the stage implements the optional contract—a versioned
|
||||||
|
semantic-configuration fingerprint. Artifact
|
||||||
records may include optional contract and external provenance objects; old
|
records may include optional contract and external provenance objects; old
|
||||||
manifests remain compatible when those fields are absent. A successful forced
|
manifests remain compatible when those fields are absent. A successful forced
|
||||||
rerun marks only succeeded transitive dependent session-stage records stale.
|
rerun marks only succeeded transitive dependent session-stage records stale.
|
||||||
The application owns a fixed dependency relation distinct from execution order;
|
The application owns a fixed dependency relation distinct from execution order;
|
||||||
dependents are returned in canonical order. Render and extract therefore never
|
dependents are returned in canonical order. Render and extract therefore never
|
||||||
stale one another, while either can stale analyze, publish, and notify.
|
stale one another, while either can stale analyze, publish, notify, and cleanup.
|
||||||
|
|
||||||
Starting an execution clears the current session-stage record's prior outputs,
|
Starting an execution clears the current session-stage record's prior outputs,
|
||||||
logs, generated configuration references, metadata, and semantic fingerprint.
|
logs, generated configuration references, metadata, and semantic fingerprint.
|
||||||
@@ -184,40 +230,36 @@ Failed and skipped
|
|||||||
transitions enforce the same clearing rule directly, while success repopulates
|
transitions enforce the same clearing rule directly, while success repopulates
|
||||||
only fields returned by the new result. Marking a record stale does not clear
|
only fields returned by the new result. Marking a record stale does not clear
|
||||||
those details because resume validation and diagnosis may still require them
|
those details because resume validation and diagnosis may still require them
|
||||||
before execution begins. Invocation run manifests remain immutable audit
|
before execution begins.
|
||||||
records of their own outcomes.
|
|
||||||
|
|
||||||
Aggregate lifecycle clearing deliberately preserves the analyze-owned
|
Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
|
||||||
per-artifact collection. This lets later reconciliation replace only evaluated
|
speaker collection and the analyze-owned per-artifact collection. This lets
|
||||||
entries without erasing unrelated current results. Other stages retain their
|
later reconciliation replace only non-current entries without erasing durable
|
||||||
existing aggregate-only lifecycle behavior and are forbidden from carrying the
|
completions. Other stages retain aggregate-only lifecycle behavior and are
|
||||||
analyze-specific fields.
|
forbidden from carrying either collection.
|
||||||
|
|
||||||
A stage may explicitly return a skipped disposition and stable reason. The
|
A stage may explicitly return a skipped disposition and stable reason. The
|
||||||
runner persists that outcome in both manifests, clears older outputs for the
|
runner persists that outcome in the session manifest, clears older outputs,
|
||||||
session-stage record along with older logs, generated configuration references,
|
logs, generated configuration references, and metadata, then applies any
|
||||||
and metadata, then applies any bounded details from the current skip and
|
bounded details from the current skip and continues. This self-skip is distinct from deciding not to execute an
|
||||||
continues. This self-skip is distinct from deciding not to execute an
|
|
||||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||||
cannot contain outputs. An intentional self-skip records the current semantic
|
cannot contain outputs. An intentional self-skip records the current semantic
|
||||||
fingerprint because it is a completed, reusable stage result; failed or
|
fingerprint because it is a completed, reusable stage result; failed or
|
||||||
interrupted work never promotes one.
|
interrupted work never promotes one.
|
||||||
|
|
||||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
When an already-succeeded stage is skipped, the session manifest deliberately
|
||||||
the `skip` action and reason. The session manifest deliberately retains its
|
retains its existing succeeded record because it remains the cross-invocation
|
||||||
existing succeeded record because it remains the cross-invocation progress
|
progress authority. If a stage supplies semantic configuration evidence, reuse first
|
||||||
authority. If a stage supplies semantic configuration evidence, reuse first
|
|
||||||
requires the persisted positive schema version and lowercase SHA-256 digest to
|
requires the persisted positive schema version and lowercase SHA-256 digest to
|
||||||
match the current resolved stage semantics. Missing legacy evidence, malformed
|
match the current resolved stage semantics. Missing semantic evidence, malformed
|
||||||
evidence, or a mismatch makes the stage and its fixed transitive dependents
|
evidence, or a mismatch makes the stage and its fixed transitive dependents
|
||||||
stale. The invocation skip copies the matched fingerprint for provenance but
|
stale. The existing stage-specific resume
|
||||||
does not rewrite session authority. The existing stage-specific resume
|
|
||||||
validator runs only after this semantic check succeeds; both checks are
|
validator runs only after this semantic check succeeds; both checks are
|
||||||
required. Extraction and analyze have resume validators and may reject an
|
required. Extraction, analyze, and cleanup have resume validators and may
|
||||||
otherwise eligible skip when their selected durable evidence is obsolete; the
|
reject an otherwise eligible skip when their selected durable evidence is
|
||||||
runner marks the aggregate record stale and executes it. Analyze's validator
|
obsolete; the runner marks the aggregate record stale and executes it.
|
||||||
can still accept a partial selection when only unrelated artifact records are
|
Analyze's validator can still accept a partial selection when only unrelated
|
||||||
stale.
|
artifact records are stale.
|
||||||
|
|
||||||
Implemented reuse coverage is deliberately split between aggregate semantic
|
Implemented reuse coverage is deliberately split between aggregate semantic
|
||||||
evidence and focused durable validators:
|
evidence and focused durable validators:
|
||||||
@@ -225,7 +267,7 @@ evidence and focused durable validators:
|
|||||||
| Work | Reuse authority | Focused owners |
|
| Work | Reuse authority | Focused owners |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
|
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
|
||||||
| transcribe | aggregate semantic fingerprint | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
|
| transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
|
||||||
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
|
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
|
||||||
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
|
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
|
||||||
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
|
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
|
||||||
@@ -235,6 +277,7 @@ evidence and focused durable validators:
|
|||||||
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
|
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
|
||||||
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
|
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
|
||||||
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
|
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
|
||||||
|
| cleanup | aggregate semantic fingerprint plus publish-evidence and target-absence validator | [cleanup](stage-cleanup.md), [workspace](workspace.md) |
|
||||||
|
|
||||||
These contracts record resolved choices Narratio can observe, not operational
|
These contracts record resolved choices Narratio can observe, not operational
|
||||||
runner tuning. External model, module, prompt, profile, and configuration-file
|
runner tuning. External model, module, prompt, profile, and configuration-file
|
||||||
@@ -242,10 +285,8 @@ contents that a tool privately loads remain outside the contract when their
|
|||||||
configured identifier is unchanged; operators must force the affected work
|
configured identifier is unchanged; operators must force the affected work
|
||||||
after such a private content change.
|
after such a private content change.
|
||||||
|
|
||||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
The session manifest is the authoritative stage-progress ledger across
|
||||||
Run manifest is invocation-scoped audit state.
|
invocations. It retains the most recently resolved invocation's bounded
|
||||||
|
|
||||||
Both manifests retain the most recently resolved invocation's bounded
|
|
||||||
configuration provenance. It identifies the selected profile name and source
|
configuration provenance. It identifies the selected profile name and source
|
||||||
(`default` or `cli`) plus the effective configuration digest, but never a raw
|
(`default` or `cli`) plus the effective configuration digest, but never a raw
|
||||||
secret or profile content. This provenance is informational: it does not
|
secret or profile content. This provenance is informational: it does not
|
||||||
@@ -254,9 +295,9 @@ invalidates only stages whose semantic configuration changed. When a private
|
|||||||
external-tool model, module, prompt, or profile changes behind an unchanged
|
external-tool model, module, prompt, or profile changes behind an unchanged
|
||||||
configured identifier, use `--force` for the affected work.
|
configured identifier, use `--force` for the affected work.
|
||||||
|
|
||||||
`session plan` computes the same current fingerprint and applies the same
|
`session plan` computes the same current fingerprint and applies the shared
|
||||||
comparison and invalidation rules to a cloned manifest. It predicts the runner
|
decision and invalidation rules to a cloned manifest. It predicts the runner
|
||||||
decision without persisting session or invocation state. The shared helper
|
decision without persisting session state. The semantic fingerprint owner
|
||||||
hashes deterministic JSON from stage-owned typed structs; stage providers must
|
hashes deterministic JSON from stage-owned typed structs; stage providers must
|
||||||
exclude secrets, complete effective-configuration dumps, and operational
|
exclude secrets, complete effective-configuration dumps, and operational
|
||||||
values that cannot affect canonical results. Concrete coverage is owned by the
|
values that cannot affect canonical results. Concrete coverage is owned by the
|
||||||
@@ -266,7 +307,7 @@ Before an explicitly bounded execution starts after `prepare`, the application
|
|||||||
reads the session manifest and accepts only `succeeded` or `skipped` for every
|
reads the session manifest and accepts only `succeeded` or `skipped` for every
|
||||||
excluded canonical prefix stage. The first other status or absent record fails
|
excluded canonical prefix stage. The first other status or absent record fails
|
||||||
the request before layout mutation, adapter initialization, session-manifest
|
the request before layout mutation, adapter initialization, session-manifest
|
||||||
writes, or run-manifest creation. Excluded prefix records are not passed to
|
writes, or run-directory creation. Excluded prefix records are not passed to
|
||||||
resume validators. Records after the selected end are not prerequisites and
|
resume validators. Records after the selected end are not prerequisites and
|
||||||
may be made stale by selected work without being scheduled.
|
may be made stale by selected work without being scheduled.
|
||||||
|
|
||||||
@@ -283,34 +324,32 @@ session manifest must agree on campaign and session identity before execution;
|
|||||||
the current projection is refreshed for every invocation while stage progress,
|
the current projection is refreshed for every invocation while stage progress,
|
||||||
inputs, and durable artifacts remain session history.
|
inputs, and durable artifacts remain session history.
|
||||||
|
|
||||||
For handled failures after an invocation record is created, the runner records
|
For handled failures after the session identity transition, the runner records
|
||||||
the failure on the session ledger and persists it before persisting the failed
|
the failure on the session ledger and atomically attempts its save while
|
||||||
run audit record. This preserves the resume authority while making a partial
|
preserving the originating error. Abrupt process death remains an accepted case
|
||||||
persistence disagreement visible. Abrupt process death remains an accepted case
|
where the last session transition can require operator interpretation.
|
||||||
where a durable running record can require operator interpretation.
|
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- stage resume/skip decisions are session-manifest driven.
|
- stage resume/skip decisions are session-manifest driven.
|
||||||
- semantic fingerprint comparison precedes stage-specific resume validation.
|
- semantic fingerprint comparison precedes stage-specific resume validation.
|
||||||
- only successful and intentional-skipped results promote current semantic
|
- only successful and intentional-skipped results promote current semantic
|
||||||
evidence; invocation reuse copies evidence without replacing session state.
|
evidence.
|
||||||
- running, failed, and self-skipped stages do not retain result payloads from
|
- running, failed, and self-skipped stages do not retain result payloads from
|
||||||
an earlier success.
|
an earlier success.
|
||||||
- stale stages retain prior details until replacement execution starts.
|
- stale stages retain prior details until replacement execution starts.
|
||||||
- force reruns stale succeeded stages in the fixed dependency relation.
|
- force reruns stale succeeded stages in the fixed dependency relation.
|
||||||
- run manifest does not replace session manifest as progress authority.
|
- the session manifest is the only durable local progress ledger.
|
||||||
- remote commitment is established by a verified current pointer and remote
|
- remote commitment is established by a verified current pointer and remote
|
||||||
commit relationship, never by a mutable session-manifest boolean.
|
commit relationship, never by a mutable session-manifest boolean.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Models and transitions: `internal/manifest/manifest.go`,
|
- Models and transitions: `internal/manifest/manifest.go`
|
||||||
`internal/manifest/run_manifest.go`
|
|
||||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||||
`internal/artifacts/current_state_commit.go`,
|
`internal/artifacts/current_state_commit.go`,
|
||||||
`internal/artifacts/current_state_legacy.go`
|
|
||||||
- Persistence and validation: `internal/manifest/store.go`
|
- Persistence and validation: `internal/manifest/store.go`
|
||||||
|
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
|
||||||
- Package tests: `internal/manifest/*_test.go`
|
- Package tests: `internal/manifest/*_test.go`
|
||||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||||
`internal/app/run_stage_test.go`
|
`internal/app/run_stage_test.go`
|
||||||
|
|||||||
@@ -22,20 +22,29 @@ session safety controls, and runs commands. Pipeline commands execute the
|
|||||||
canonical stage sequence through adapter interfaces, while manifests record
|
canonical stage sequence through adapter interfaces, while manifests record
|
||||||
progress and artifact services resolve durable inputs and outputs.
|
progress and artifact services resolve durable inputs and outputs.
|
||||||
|
|
||||||
|
The pipeline runner follows one explicit invocation lifecycle: inspect bounded
|
||||||
|
prerequisites without mutation, prepare the workspace layout, acquire one
|
||||||
|
session lock, load or create the authoritative session manifest, compose only
|
||||||
|
the collaborators needed by the selected range, execute each selected stage in
|
||||||
|
order, and then return the summary. Cleanup, when selected, is the ordinary
|
||||||
|
final stage rather than runner finalization work.
|
||||||
|
The lock remains held from manifest initialization through finalization, and a
|
||||||
|
failed stage or required manifest save prevents later stages from starting.
|
||||||
|
|
||||||
## Components
|
## Components
|
||||||
|
|
||||||
| Area | Implemented owners | Responsibility |
|
| Area | Implemented owners | Responsibility |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
|
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||||
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||||
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
|
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
|
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus atomic single-file replacement, bounded reads, workspace modes, and root-scoped cleanup helpers. |
|
||||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||||
|
|
||||||
The application boundary composes concrete implementations. Stages depend on
|
The application boundary composes concrete implementations. Stages depend on
|
||||||
@@ -43,10 +52,20 @@ Narratio-level contracts; external transport and SDK details remain in
|
|||||||
adapters. The normative rules for these relationships remain in
|
adapters. The normative rules for these relationships remain in
|
||||||
[Architecture](../policy/architecture.md).
|
[Architecture](../policy/architecture.md).
|
||||||
|
|
||||||
|
Command execution normalizes one private dependency value per invocation.
|
||||||
|
Pipeline configuration-path discovery, configuration reads, secret-directory
|
||||||
|
reads, remote downloads, object-store construction, and restore phase functions
|
||||||
|
use that value throughout the command. Public command entry points supply the
|
||||||
|
canonical discovery and production defaults, while focused tests may provide
|
||||||
|
invocation-owned discovery candidates or replace only the external or sequencing
|
||||||
|
boundary they exercise without mutating package state.
|
||||||
|
|
||||||
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
||||||
model. Planning clones session state and applies selected-stage transitions and
|
model and the same pure pre-execution decision over recorded status, force, and
|
||||||
resume validation in memory; it does not create invocation state or initialize
|
normalized resume validation. Planning applies that decision and modeled
|
||||||
stage-execution adapters. Command configuration loading can still retrieve a
|
selected-stage transitions to cloned session state; it does not persist, create
|
||||||
|
invocation state, initialize stage-execution adapters, or claim to observe an
|
||||||
|
execution result. Command configuration loading can still retrieve a
|
||||||
missing session file through configured remote storage. It retains the initially
|
missing session file through configured remote storage. It retains the initially
|
||||||
composed pipeline and selected campaign while resolving either a local or
|
composed pipeline and selected campaign while resolving either a local or
|
||||||
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
||||||
@@ -68,6 +87,7 @@ The implemented canonical order is:
|
|||||||
9. [`analyze`](stage-analyze.md)
|
9. [`analyze`](stage-analyze.md)
|
||||||
10. [`publish`](stage-publish.md)
|
10. [`publish`](stage-publish.md)
|
||||||
11. `notify` (no-op)
|
11. `notify` (no-op)
|
||||||
|
12. [`cleanup`](stage-cleanup.md)
|
||||||
|
|
||||||
`notify` currently has no persisted pipeline outputs and uses the explicit
|
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||||
`noop` notification mode. Its versioned semantic evidence records that delivery
|
`noop` notification mode. Its versioned semantic evidence records that delivery
|
||||||
@@ -81,18 +101,19 @@ registry owns the flat execution sequence. The application orchestration owner
|
|||||||
uses a fixed, validated dependency relation to find transitive dependents in
|
uses a fixed, validated dependency relation to find transitive dependents in
|
||||||
canonical order. In particular, `render` and `extract` are sibling consumers of
|
canonical order. In particular, `render` and `extract` are sibling consumers of
|
||||||
trimmed transcript state: neither invalidates the other, while either can stale
|
trimmed transcript state: neither invalidates the other, while either can stale
|
||||||
`analyze`, `publish`, and `notify`.
|
`analyze`, `publish`, `notify`, and `cleanup`. Cleanup is a dependent of notify,
|
||||||
|
so any earlier replacement can stale a prior cleanup result.
|
||||||
|
|
||||||
## Focused Documentation
|
## Focused Documentation
|
||||||
|
|
||||||
- [Configuration Internals](configuration.md): pipeline composition, import
|
- [Configuration Internals](configuration.md): pipeline imports and profiles,
|
||||||
confinement, field ownership, decoding, and root-relative path semantics.
|
field ownership, decoding, and root-relative path semantics.
|
||||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||||
failure behavior, and test surfaces.
|
failure behavior, and test surfaces.
|
||||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||||
resolution, previous requirements, and current-state helpers.
|
resolution, previous requirements, and current-state helpers.
|
||||||
- [Manifest Internals](manifest.md): session and run records, persistence, and
|
- [Manifest Internals](manifest.md): session records, persistence, and execution
|
||||||
execution transitions.
|
transitions.
|
||||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||||
guardrails.
|
guardrails.
|
||||||
@@ -108,6 +129,7 @@ trimmed transcript state: neither invalidates the other, while either can stale
|
|||||||
- [`extract`](stage-extract.md)
|
- [`extract`](stage-extract.md)
|
||||||
- [`analyze`](stage-analyze.md)
|
- [`analyze`](stage-analyze.md)
|
||||||
- [`publish`](stage-publish.md)
|
- [`publish`](stage-publish.md)
|
||||||
|
- [`cleanup`](stage-cleanup.md)
|
||||||
|
|
||||||
Use this map to find an owner, then read its focused documentation and tests
|
Use this map to find an owner, then read its focused documentation and tests
|
||||||
before changing behavior.
|
before changing behavior.
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ dependency order, and safely materialize validated outputs.
|
|||||||
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
||||||
party artifact families have already expanded into this map during
|
party artifact families have already expanded into this map during
|
||||||
configuration resolution, including corresponding member dependencies and
|
configuration resolution, including corresponding member dependencies and
|
||||||
rewritten member-artifact input sources
|
rewritten member-artifact input sources. Each artifact's dependency list is
|
||||||
|
already the normalized union of explicit ordering edges and configured-artifact
|
||||||
|
input edges.
|
||||||
- optional selected artifact keys supplied through the stage environment
|
- optional selected artifact keys supplied through the stage environment
|
||||||
- built-in, configured, extraction, and previous-session source references in
|
- built-in, configured, extraction, and previous-session source references in
|
||||||
artifact inputs
|
artifact inputs
|
||||||
@@ -31,12 +33,14 @@ Supported source families:
|
|||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- when `pipeline.scriptorium` is absent or no configured artifact is
|
- when `pipeline.scriptorium` is absent or no configured artifact is
|
||||||
executable, completes successfully with no outputs and records explanatory
|
executable, completes successfully and records explanatory metadata. Before
|
||||||
metadata. This is not an explicit self-skip: both manifests record success,
|
returning, it reconciles and checkpoints the complete session collection:
|
||||||
satisfy publish's prerequisite, and an ordinary later run reuses the result
|
records removed from configuration are dropped, while valid records for
|
||||||
while the effective set remains empty. Enabling or selecting an artifact
|
configured but unselected artifacts remain authoritative. This is not an
|
||||||
later makes missing versioned evidence non-resumable and schedules it without
|
explicit self-skip: the session manifest records success, satisfies publish's
|
||||||
requiring force.
|
prerequisite, and an ordinary later run reuses the result while the effective
|
||||||
|
set remains empty. Enabling or selecting an artifact later makes missing
|
||||||
|
versioned evidence non-resumable and schedules it without requiring force.
|
||||||
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||||
and configured extraction lanes. Extraction availability is hydrated only
|
and configured extraction lanes. Extraction availability is hydrated only
|
||||||
from compatible successful extraction evidence.
|
from compatible successful extraction evidence.
|
||||||
@@ -49,6 +53,9 @@ Supported source families:
|
|||||||
manifest record and durable output pass the configured-artifact evidence
|
manifest record and durable output pass the configured-artifact evidence
|
||||||
contract; an incidental or stale canonical file is unavailable.
|
contract; an incidental or stale canonical file is unavailable.
|
||||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||||
|
- consumes only the concrete dependency graph finalized by configuration;
|
||||||
|
planning, prerequisite closure, execution, reuse, invalidation, and semantic
|
||||||
|
evidence do not infer a second graph from inputs.
|
||||||
- resolves required/optional inputs per artifact source definition into an
|
- resolves required/optional inputs per artifact source definition into an
|
||||||
ordered semantic identity. Each identity records the configured input name,
|
ordered semantic identity. Each identity records the configured input name,
|
||||||
canonical source ID, required policy, explicit presence, source contract,
|
canonical source ID, required policy, explicit presence, source contract,
|
||||||
@@ -75,7 +82,7 @@ Supported source families:
|
|||||||
validated current dependency-output identities, and sorted effective
|
validated current dependency-output identities, and sorted effective
|
||||||
Scriptorium variables (including Narratio's sticky session variable).
|
Scriptorium variables (including Narratio's sticky session variable).
|
||||||
- provides read-only reconciliation that classifies each configured record as
|
- provides read-only reconciliation that classifies each configured record as
|
||||||
current, stale, missing, failed, legacy, or otherwise non-resumable, and
|
current, stale, missing, failed, or otherwise non-resumable, and
|
||||||
separately identifies manifest records removed from current configuration.
|
separately identifies manifest records removed from current configuration.
|
||||||
A record is current only when its fingerprint version and value match and its
|
A record is current only when its fingerprint version and value match and its
|
||||||
configured output still passes manifest-authoritative evidence validation.
|
configured output still passes manifest-authoritative evidence validation.
|
||||||
@@ -89,13 +96,14 @@ Supported source families:
|
|||||||
rebuilt when required, while unrelated disabled artifacts are excluded.
|
rebuilt when required, while unrelated disabled artifacts are excluded.
|
||||||
- the work plan carries explicit targets, prerequisite-only work, deterministic
|
- the work plan carries explicit targets, prerequisite-only work, deterministic
|
||||||
execution and reuse lists, invalidated and removed records, and a cloned
|
execution and reuse lists, invalidated and removed records, and a cloned
|
||||||
projected record collection. Valid unrelated configured records survive the
|
reconciled starting collection. Valid unrelated configured records survive
|
||||||
projection, removed records are omitted, and legacy files never become
|
reconciliation, removed records are omitted, and incidental files never
|
||||||
current without regeneration.
|
become current without manifest-backed regeneration.
|
||||||
- implements aggregate resume validation by running the same read-only catalog,
|
- implements aggregate resume validation by running the same read-only catalog,
|
||||||
fingerprint reconciliation, and work planner used by execution. A succeeded
|
fingerprint reconciliation, and work planner used by execution. A succeeded
|
||||||
aggregate record is reusable exactly when the selected closure schedules no
|
aggregate record is reusable exactly when the selected closure schedules no
|
||||||
artifact work; stale unrelated records do not block a partial selection.
|
artifact work and no removed configuration records require a checkpoint;
|
||||||
|
stale unrelated records do not block a partial selection.
|
||||||
- exposes the typed artifact decision to `session plan`. Planning applies it to
|
- exposes the typed artifact decision to `session plan`. Planning applies it to
|
||||||
a cloned manifest after modeling earlier selected stage transitions, so
|
a cloned manifest after modeling earlier selected stage transitions, so
|
||||||
aggregate run/skip and artifact execute/reuse decisions match the ordinary
|
aggregate run/skip and artifact execute/reuse decisions match the ordinary
|
||||||
@@ -104,6 +112,10 @@ Supported source families:
|
|||||||
prerequisites remain available through the runtime catalog without invoking
|
prerequisites remain available through the runtime catalog without invoking
|
||||||
Scriptorium; newly produced prerequisites enter that catalog with the same
|
Scriptorium; newly produced prerequisites enter that catalog with the same
|
||||||
contract, checksum, and size identity used for persisted current evidence.
|
contract, checksum, and size identity used for persisted current evidence.
|
||||||
|
- before the first adapter call, checkpoints the reconciled starting collection
|
||||||
|
when it removed configuration records or cleared obsolete authority. After
|
||||||
|
each successful artifact, checkpoints the complete collection before making
|
||||||
|
that artifact available to the next scheduled dependent.
|
||||||
- keeps adapter output in the invocation's run-local analyze directory until
|
- keeps adapter output in the invocation's run-local analyze directory until
|
||||||
it is a safe, non-empty, bounded regular file with a calculated checksum and
|
it is a safe, non-empty, bounded regular file with a calculated checksum and
|
||||||
complete output contract. Canonical replacement uses the shared atomic file
|
complete output contract. Canonical replacement uses the shared atomic file
|
||||||
@@ -112,16 +124,15 @@ Supported source families:
|
|||||||
- records each successful artifact's freshly computed fingerprint, canonical
|
- records each successful artifact's freshly computed fingerprint, canonical
|
||||||
relative output path, contract, checksum, size, producer run ID, bounded
|
relative output path, contract, checksum, size, producer run ID, bounded
|
||||||
Scriptorium provenance, logs, and generated configuration references in the
|
Scriptorium provenance, logs, and generated configuration references in the
|
||||||
analyze-owned projection.
|
analyze-owned manifest collection.
|
||||||
- preserves valid unrelated current records during partial execution. If a
|
- preserves valid unrelated current records during partial execution. If a
|
||||||
rebuilt output's bytes and contract are unchanged, unselected dependents may
|
rebuilt output's bytes and contract are unchanged, unselected dependents may
|
||||||
remain current. If that semantic identity changes, unselected transitive
|
remain current. If that semantic identity changes, unselected transitive
|
||||||
dependents become stale without being executed; dependents included in the
|
dependents become stale without being executed; dependents included in the
|
||||||
invocation are evaluated in dependency order instead.
|
invocation are evaluated in dependency order instead.
|
||||||
- reports all evaluated targets and prerequisites in invocation state. The
|
- returns aggregate outputs from every current session record in normalized-key
|
||||||
runner reconstructs aggregate session outputs from every current session
|
order after the complete reconciled collection has been checkpointed;
|
||||||
record and invocation outputs from only records produced by the current run.
|
unrelated stale records do not make an otherwise successful partial
|
||||||
Unrelated stale records do not make an otherwise successful partial
|
|
||||||
invocation fail.
|
invocation fail.
|
||||||
- resolves previous-session sources from local `previous/` cache only.
|
- resolves previous-session sources from local `previous/` cache only.
|
||||||
- runs optional render-debug, then artifact execution.
|
- runs optional render-debug, then artifact execution.
|
||||||
@@ -137,25 +148,31 @@ Supported source families:
|
|||||||
guidance.
|
guidance.
|
||||||
- dependency cycles or unavailable required dependencies fail.
|
- dependency cycles or unavailable required dependencies fail.
|
||||||
- adapter validation failures fail stage.
|
- adapter validation failures fail stage.
|
||||||
- a scheduled artifact failure returns the restricted analyze-state projection
|
- a scheduled artifact failure installs a retry collection on the live manifest
|
||||||
with the active artifact marked `failed`, a bounded error, and no output
|
with the active artifact marked `failed`, a bounded error, and no output
|
||||||
authority. Current transitive dependents become stale without execution.
|
authority. Current transitive dependents become stale without execution.
|
||||||
- earlier artifacts from the invocation remain current only after their
|
- earlier artifacts remain current only after their run-local output passed
|
||||||
run-local output passed validation and canonical materialization. They remain
|
validation, canonical materialization, and a successful complete-collection
|
||||||
in invocation history; unattempted later artifacts do not appear there.
|
manifest checkpoint; unattempted later artifacts retain their reconciled
|
||||||
|
session state.
|
||||||
- unrelated current records survive a partial failure. Old canonical bytes for
|
- unrelated current records survive a partial failure. Old canonical bytes for
|
||||||
the failed artifact and newly materialized bytes whose projection cannot be
|
the failed artifact and newly materialized bytes whose checkpoint fails are
|
||||||
persisted are incidental, not current evidence.
|
incidental, not current evidence.
|
||||||
- the runner persists a valid partial projection before it marks aggregate
|
- a checkpoint failure stops the queue immediately. The in-memory retry
|
||||||
analyze failed and invalidates publish and notify through the application
|
collection starts from the last successful checkpoint, marks the active
|
||||||
dependency relation. Projection-persistence errors retain the last durable
|
artifact failed without output authority, and stales its current transitive
|
||||||
per-artifact authority and are joined with the original failure context.
|
dependents.
|
||||||
|
- the runner's ordinary error path marks aggregate analyze failed, clears its
|
||||||
|
aggregate outputs, invalidates publish, notify, and cleanup through the
|
||||||
|
application dependency relation, and persists the live retry collection.
|
||||||
|
Terminal-save errors are joined with the original artifact or checkpoint
|
||||||
|
failure.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||||
- input-identity resolution is read-only: it does not invoke adapters,
|
- input-identity resolution is read-only: it does not invoke adapters,
|
||||||
materialize outputs, update status, or create run records.
|
materialize outputs, update status, or create run-local files.
|
||||||
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
||||||
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
||||||
executable or private transitive configuration contents. A change that is
|
executable or private transitive configuration contents. A change that is
|
||||||
@@ -164,6 +181,9 @@ Supported source families:
|
|||||||
- output provenance and metadata are deterministic per execution.
|
- output provenance and metadata are deterministic per execution.
|
||||||
- a canonical file without current per-artifact manifest evidence is never
|
- a canonical file without current per-artifact manifest evidence is never
|
||||||
promoted to current state.
|
promoted to current state.
|
||||||
|
- an ordinary retry after interruption or partial failure reuses every valid
|
||||||
|
earlier checkpoint and schedules only failed, stale, missing, or otherwise
|
||||||
|
non-current work in dependency order.
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|
||||||
@@ -173,13 +193,18 @@ Supported source families:
|
|||||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||||
- Implementation and tests: `internal/stage/analyze.go`,
|
- Implementation and tests: `internal/stage/analyze.go`,
|
||||||
`internal/stage/analyze_input_identity.go`, `internal/stage/analyze_test.go`,
|
`internal/stage/analyze_artifact_execution.go`,
|
||||||
`internal/stage/analyze_input_identity_test.go`,
|
`internal/stage/analyze_execution_state.go`,
|
||||||
|
`internal/stage/analyze_input_identity.go`,
|
||||||
`internal/stage/analyze_fingerprint.go`,
|
`internal/stage/analyze_fingerprint.go`,
|
||||||
|
`internal/stage/analyze_reconciliation.go`, `internal/stage/analyze_plan.go`,
|
||||||
|
`internal/stage/analyze_resume.go`, `internal/stage/analyze_test.go`,
|
||||||
|
`internal/stage/analyze_checkpoint_test.go`,
|
||||||
|
`internal/stage/analyze_input_identity_test.go`,
|
||||||
`internal/stage/analyze_fingerprint_test.go`,
|
`internal/stage/analyze_fingerprint_test.go`,
|
||||||
`internal/stage/analyze_reconciliation.go`, and
|
|
||||||
`internal/stage/analyze_reconciliation_test.go`,
|
`internal/stage/analyze_reconciliation_test.go`,
|
||||||
`internal/stage/analyze_plan.go`, `internal/stage/analyze_plan_test.go`, and
|
`internal/stage/analyze_plan_test.go`,
|
||||||
`internal/stage/analyze_incremental_execution_test.go`, and
|
`internal/stage/analyze_incremental_execution_test.go`,
|
||||||
`internal/stage/analyze_failure_test.go`,
|
`internal/stage/analyze_failure_test.go`,
|
||||||
`internal/stage/analyze_resume.go`, and `internal/stage/analyze_resume_test.go`
|
`internal/stage/analyze_resume_test.go`, and
|
||||||
|
`internal/app/analyze_checkpoint_recovery_test.go`.
|
||||||
|
|||||||
54
docs/internal/stage-cleanup.md
Normal file
54
docs/internal/stage-cleanup.md
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
# Stage: cleanup
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Remove explicitly configured local data for the run selected by a successful
|
||||||
|
publish commit. Cleanup is the twelfth and final canonical stage, after
|
||||||
|
`notify`.
|
||||||
|
|
||||||
|
## Authorization And Targets
|
||||||
|
|
||||||
|
Cleanup is disabled unless at least one of these policies is true:
|
||||||
|
|
||||||
|
- `pipeline.spool.delete_audio_after_publish`
|
||||||
|
- `pipeline.workspace.cleanup_after_publish`
|
||||||
|
|
||||||
|
An enabled cleanup requires the manifest's publish stage to be succeeded with
|
||||||
|
`uploaded=true`, a canonical `published_run_id`, and the exact remote commit and
|
||||||
|
current commit-pointer keys for the manifest campaign/session prefix. The
|
||||||
|
configured campaign and session must match the manifest. Cleanup derives its
|
||||||
|
only possible targets through the canonical artifact path helpers:
|
||||||
|
|
||||||
|
- the published run's session spool-audio directory; and
|
||||||
|
- the published run's workspace run directory.
|
||||||
|
|
||||||
|
It does not scan parent directories, infer the newest run, or read object
|
||||||
|
storage. Every deletion uses the root-confined file operation, which rejects a
|
||||||
|
root itself, traversal, symlink escape, and out-of-root targets.
|
||||||
|
|
||||||
|
## Execution And Resume
|
||||||
|
|
||||||
|
Disabled cleanup records an explicit skipped result. Enabled cleanup removes
|
||||||
|
targets in deterministic policy order and treats an already-missing target as
|
||||||
|
success. A successful record contains the published run ID and completed policy
|
||||||
|
names.
|
||||||
|
|
||||||
|
A previous cleanup success is reusable only while its semantic configuration
|
||||||
|
matches and every selected target remains absent. A recreated target makes the
|
||||||
|
stage non-resumable. A failure is recorded against cleanup; publish remains
|
||||||
|
succeeded and remote current state is unchanged. On a later normal full run,
|
||||||
|
valid upstream stages are skipped and cleanup is retried without republishing.
|
||||||
|
|
||||||
|
`narratio publish` selects publish only and never invokes cleanup. A bounded run
|
||||||
|
through `notify` or any earlier endpoint also excludes it. Operators may run
|
||||||
|
`narratio run-stage cleanup <session_id>` directly; missing, failed, malformed,
|
||||||
|
or identity-mismatched publication evidence fails before deletion.
|
||||||
|
|
||||||
|
## Implementation And Tests
|
||||||
|
|
||||||
|
- Planning and evidence validation: `internal/stage/cleanup_plan.go`
|
||||||
|
- Execution and resume: `internal/stage/cleanup.go`
|
||||||
|
- Root-confined deletion: `internal/fileops/cleanup.go`
|
||||||
|
- Focused tests: `internal/stage/cleanup_plan_test.go`,
|
||||||
|
`internal/stage/cleanup_test.go`, and
|
||||||
|
`internal/app/cleanup_orchestration_test.go`
|
||||||
@@ -19,12 +19,17 @@ procedures belong in [Operations](../operations.md).
|
|||||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||||
2. resolves every configured prepared reference through the shared
|
2. resolves every configured prepared reference through the shared
|
||||||
manifest-authoritative identity resolver before creating run-local output;
|
manifest-authoritative identity resolver before creating run-local output;
|
||||||
3. streams each verified reference into an invocation-local snapshot and
|
3. resolves each reference to a stable content-addressed session snapshot,
|
||||||
rejects any source change observed while copying;
|
reusing verified bytes or reconstructing a missing/corrupt regular entry,
|
||||||
|
and rejects any source change observed while copying;
|
||||||
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
||||||
including sorted reference identities;
|
including sorted reference identities and the explicit downstream
|
||||||
|
configuration-profile selection;
|
||||||
5. creates a run-local staging directory and invokes the injected
|
5. creates a run-local staging directory and invokes the injected
|
||||||
`notarius.Runner`;
|
`notarius.Runner`, carrying the optional already-resolved credential
|
||||||
|
reference without exposing its value, the optional exact configuration
|
||||||
|
profile, plus effective resume and chunk-cache controls after force
|
||||||
|
precedence;
|
||||||
6. revalidates the reference snapshots, then validates the v2 successful
|
6. revalidates the reference snapshots, then validates the v2 successful
|
||||||
receipt, confined index, management documents, configured required lane
|
receipt, confined index, management documents, configured required lane
|
||||||
descriptors, validation summaries, and regular payload files;
|
descriptors, validation summaries, and regular payload files;
|
||||||
@@ -41,13 +46,21 @@ Narratio run ID, the resolved trimmed-input identity, and invocation
|
|||||||
fingerprint. The input identity binds the exact transcript bytes, canonical
|
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||||
source ID, producer stage/output/run identity, and resolution provenance.
|
source ID, producer stage/output/run identity, and resolution provenance.
|
||||||
Reference metadata contains only selector, source ID, canonical session-relative
|
Reference metadata contains only selector, source ID, canonical session-relative
|
||||||
path, checksum, and size; adapter requests receive selector and absolute
|
path, checksum, and size; adapter requests receive selector and absolute stable
|
||||||
invocation-local snapshot path, never payload contents. Snapshot bytes must
|
snapshot path, never payload contents. Snapshot bytes must
|
||||||
match the prepared identity both before and after Notarius runs, so a concurrent
|
match the prepared identity both before and after Notarius runs, so a concurrent
|
||||||
prepared-file replacement cannot make recorded provenance describe different
|
prepared-file replacement cannot make recorded provenance describe different
|
||||||
bytes from those supplied to Notarius.
|
bytes from those supplied to Notarius.
|
||||||
Validation completes before
|
Validation completes before the single durable rename, so a rejected result
|
||||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
cannot expose a partial durable bundle.
|
||||||
|
|
||||||
|
Snapshot destinations are owned by `internal/artifacts` under
|
||||||
|
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
|
||||||
|
or timestamp and can be reused across invocations. Identical bytes and basename
|
||||||
|
share a verified file; changed bytes select a different digest directory.
|
||||||
|
Unsafe path components and non-regular conflicts fail without recursive
|
||||||
|
deletion. Snapshot paths are delivery details and never become manifest output
|
||||||
|
evidence or prerequisites for reuse of an already complete extract bundle.
|
||||||
|
|
||||||
Any executed extraction outcome that replaces a different effective outcome
|
Any executed extraction outcome that replaces a different effective outcome
|
||||||
marks succeeded analysis and delivery dependents stale. Render is an independent
|
marks succeeded analysis and delivery dependents stale. Render is an independent
|
||||||
@@ -59,17 +72,28 @@ outputs is stable and does not repeatedly invalidate dependent stages.
|
|||||||
Before the focused validator runs, the application compares extract's versioned
|
Before the focused validator runs, the application compares extract's versioned
|
||||||
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
||||||
reference selector/source mappings, sorted declared output contracts, and each
|
reference selector/source mappings, sorted declared output contracts, and each
|
||||||
canonical `narratio.extraction.<key>` output identity. It excludes executable,
|
canonical `narratio.extraction.<key>` output identity. It also covers the
|
||||||
|
explicit Notarius configuration-profile selection; omitted, empty, and exact
|
||||||
|
`default` selectors share the base-selection identity. It excludes executable,
|
||||||
timeout, working directory, config path, and private Notarius config contents.
|
timeout, working directory, config path, and private Notarius config contents.
|
||||||
|
Credential presence, reference name, and resolved value are likewise
|
||||||
|
operational-only and excluded. Ordinary command loading still resolves a
|
||||||
|
selected reference before resume evaluation, but a credential-only change does
|
||||||
|
not rerun a valid completed extraction or stale its dependents.
|
||||||
|
|
||||||
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
||||||
stage record still matches the current byte- and provenance-bearing invocation
|
stage record still matches the current byte- and provenance-bearing invocation
|
||||||
evidence. That evidence covers the current direct trimmed-transcript identity,
|
evidence. That evidence covers the current direct trimmed-transcript identity,
|
||||||
sorted prepared-reference identities, pipeline identity, and configured output
|
sorted prepared-reference identities, pipeline identity, and configured output
|
||||||
contracts. The same reference helper and transcript identity are resolved again
|
contracts, plus the same downstream profile-selection identity. The same
|
||||||
for artifact evidence, so changing current transcript bytes, reference bytes,
|
reference helper and transcript identity are resolved again for artifact
|
||||||
or producer identity makes the prior extraction obsolete. Operational runner
|
evidence, so changing current transcript bytes, reference bytes, producer
|
||||||
settings do not invalidate otherwise current durable evidence.
|
identity, or a named downstream selection makes the prior extraction obsolete.
|
||||||
|
Operational runner settings do not invalidate otherwise current durable
|
||||||
|
evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
|
||||||
|
make a complete extraction non-resumable. If extraction executes, ordinary
|
||||||
|
runs forward the configured values, while force disables resume and selects
|
||||||
|
`refresh`.
|
||||||
|
|
||||||
A valid prepared-reference change makes extraction non-resumable. Missing,
|
A valid prepared-reference change makes extraction non-resumable. Missing,
|
||||||
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
||||||
@@ -82,27 +106,29 @@ contracts and provenance, regular-file status, and stored checksums. Missing or
|
|||||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||||
return an error rather than silently accepting or replacing data.
|
return an error rather than silently accepting or replacing data.
|
||||||
|
|
||||||
Neither contract can observe files imported by Notarius configuration, profile
|
This sensitivity is limited to the explicit downstream selector. Narratio's
|
||||||
contents, prompt/module definitions, or other transitive inputs. Operators must
|
own profile label and whole effective-configuration digest are not blanket
|
||||||
force extraction after changing any such private input behind a stable
|
extraction invalidators. Neither contract can observe files imported by
|
||||||
identifier.
|
Notarius configuration, profile contents, prompt/module definitions, or other
|
||||||
|
transitive inputs. Operators must force extraction after changing any such
|
||||||
|
private input behind a stable identifier.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||||
index compatibility, inconsistent warning or diagnostic envelopes,
|
index compatibility, inconsistent warning or diagnostic envelopes,
|
||||||
required-lane rejection or incomplete validation, payload inspection,
|
required-lane rejection or incomplete validation, payload inspection,
|
||||||
checksum, or promotion errors fail the stage through ordinary manifest
|
checksum, or installation errors fail the stage through ordinary manifest
|
||||||
transition handling.
|
transition handling.
|
||||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||||
not given selectable extraction sources unless the complete configured result
|
not given selectable extraction sources unless the complete configured result
|
||||||
has passed validation and promotion.
|
has passed validation and installation.
|
||||||
|
|
||||||
When a replacement attempt begins, the current session-stage record no longer
|
When a replacement attempt begins, the current session-stage record no longer
|
||||||
advertises payload from the previous success. A failed replacement therefore
|
advertises payload from the previous success. A failed replacement therefore
|
||||||
has no current outputs, logs, generated configuration references, or metadata,
|
has no current outputs, logs, generated configuration references, or metadata,
|
||||||
while the earlier invocation manifest and immutable promoted bundle remain
|
while the earlier immutable promoted bundle remains available for audit and
|
||||||
available for audit and recovery.
|
recovery.
|
||||||
|
|
||||||
## Implementation And Focused Tests
|
## Implementation And Focused Tests
|
||||||
|
|
||||||
@@ -110,9 +136,13 @@ available for audit and recovery.
|
|||||||
`internal/stage/extract_resume.go`,
|
`internal/stage/extract_resume.go`,
|
||||||
`internal/stage/extract_test.go`,
|
`internal/stage/extract_test.go`,
|
||||||
`internal/stage/semantic_contracts_delivery.go`
|
`internal/stage/semantic_contracts_delivery.go`
|
||||||
|
- Stable snapshot materialization and verification:
|
||||||
|
`internal/stage/extract_references.go`
|
||||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||||
`internal/adapters/notarius/subprocess_test.go`
|
`internal/adapters/notarius/subprocess_test.go`
|
||||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||||
`internal/artifacts/extraction_catalog_test.go`
|
`internal/artifacts/extraction_catalog_test.go`
|
||||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||||
|
- Opt-in Notarius v0.7.0 production-boundary coverage:
|
||||||
|
`internal/stage/notarius_cache_integration_test.go`
|
||||||
|
|||||||
@@ -17,7 +17,11 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- discovers and validates raw transcript inputs.
|
- discovers and validates raw transcript inputs. For versioned transcribe
|
||||||
|
state, discovery requires aggregate transcribe success and consumes only the
|
||||||
|
aggregate outputs that exactly match the current per-speaker collection.
|
||||||
|
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
|
||||||
|
workflows without versioned speaker state retain directory discovery.
|
||||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||||
- validates merged transcript and optional report JSON.
|
- validates merged transcript and optional report JSON.
|
||||||
@@ -26,6 +30,8 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- merge always consumes normalized forms of raw inputs.
|
- merge always consumes normalized forms of raw inputs.
|
||||||
|
- a versioned transcribe collection is usable only through its complete,
|
||||||
|
successful aggregate output declaration.
|
||||||
- base transcript must validate before stage success.
|
- base transcript must validate before stage success.
|
||||||
- report output is config-gated.
|
- report output is config-gated.
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ Normalize polished transcript into final transcript using Seriatim.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
- resolves polished transcript only from verified manifest output evidence.
|
||||||
- applies `pipeline.normalize` config or default normalize config.
|
- applies `pipeline.normalize` config or default normalize config.
|
||||||
- runs Seriatim normalize with configured timeout/binary.
|
- runs Seriatim normalize with configured timeout/binary.
|
||||||
- validates normalized transcript and optional report.
|
- validates normalized transcript and optional report.
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ Run Audita polishing on base transcript and produce polished transcript.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- resolves base transcript from merge outputs/canonical fallback.
|
- resolves base transcript only from verified merge manifest output evidence.
|
||||||
- invokes an Audita runner configured with static model/runtime options; the
|
- invokes an Audita runner configured with static model/runtime options; the
|
||||||
invocation supplies paths and modules.
|
invocation supplies paths and modules.
|
||||||
- validates processed transcript structure (`segments` array required).
|
- validates processed transcript structure (`segments` array required).
|
||||||
@@ -35,8 +35,9 @@ validation model, module set, transcript description, output schema, selected
|
|||||||
external configuration path, and canonical polished-transcript identity. Module
|
external configuration path, and canonical polished-transcript identity. Module
|
||||||
ordering is normalized because the configured modules form a set. Audita's
|
ordering is normalized because the configured modules form a set. Audita's
|
||||||
executable, timeouts, concurrency, report and debug behavior, work retention,
|
executable, timeouts, concurrency, report and debug behavior, work retention,
|
||||||
and credential environment name are operational and do not invalidate a
|
and resolved credential value are operational and do not invalidate a
|
||||||
successful result.
|
successful result. The public credential reference name may participate in
|
||||||
|
configuration provenance but never exposes its resolved value.
|
||||||
|
|
||||||
Narratio can fingerprint a selected model, module, or configuration identifier,
|
Narratio can fingerprint a selected model, module, or configuration identifier,
|
||||||
but it cannot inspect content that Audita privately resolves behind that stable
|
but it cannot inspect content that Audita privately resolves behind that stable
|
||||||
|
|||||||
@@ -39,9 +39,8 @@ Materialize canonical current-session inputs before processing stages.
|
|||||||
- materializes a configured spell catalog with checksum and provenance, or
|
- materializes a configured spell catalog with checksum and provenance, or
|
||||||
safely removes an obsolete canonical spell catalog and its manifest record
|
safely removes an obsolete canonical spell catalog and its manifest record
|
||||||
when the effective input is omitted.
|
when the effective input is omitted.
|
||||||
- in canonical party mode, copies the validated raw party bytes unchanged and
|
- copies the validated canonical party bytes unchanged and deterministically
|
||||||
deterministically generates the prepared players projection; legacy mode
|
generates the prepared players projection.
|
||||||
continues to copy its opaque party and explicit players sources.
|
|
||||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||||
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||||
- resolves the pointer-selected previous source through the shared resolver;
|
- resolves the pointer-selected previous source through the shared resolver;
|
||||||
@@ -49,8 +48,10 @@ Materialize canonical current-session inputs before processing stages.
|
|||||||
- records previous inputs in `manifest.inputs`.
|
- records previous inputs in `manifest.inputs`.
|
||||||
|
|
||||||
Required previous-session inputs fail when unavailable; optional missing inputs
|
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||||
are typed skipped results. Committed sources use their exact source-to-destination
|
are typed skipped results. Each requirement resolves only from its unique
|
||||||
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
|
`published_output` declaration in the pointer-selected remote commit. Prepare
|
||||||
|
strips that selected run's immutable `published/` prefix to derive the local
|
||||||
|
cache path, then validates the committed checksum, size, and generation.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
|
|||||||
@@ -7,15 +7,16 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
|||||||
## Inputs
|
## Inputs
|
||||||
|
|
||||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||||
- invocation-scoped run files
|
- declared run-local diagnostic files
|
||||||
- resolved publish output rules
|
- resolved publish output rules
|
||||||
- effective publish locks (static + remote merged lock set), revalidated at the
|
- independent per-source remote lock objects read at planning start
|
||||||
remote commit boundary
|
- the validated prior current commit when an initially locked source may carry
|
||||||
- durable previous-session cache files when present
|
- durable previous-session cache files when present
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
- uploaded invocation record and selected publish outputs;
|
- uploaded session manifest, declared run-local diagnostics, and selected
|
||||||
|
publish outputs;
|
||||||
- uploaded durable previous-session cache files when present;
|
- uploaded durable previous-session cache files when present;
|
||||||
- immutable run-scoped commit manifest; and
|
- immutable run-scoped commit manifest; and
|
||||||
- current commit pointer, written last.
|
- current commit pointer, written last.
|
||||||
@@ -25,14 +26,27 @@ Exact remote placement and the operator workflow belong in
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
|
`publish.go` retains the transaction-owning `Run` path. Source discovery,
|
||||||
|
manifest-derived archive metadata, confined opening, and checksum validation
|
||||||
|
live in `publish_sources.go`; artifact-catalog matching, selection, required
|
||||||
|
and optional rules, destination resolution, and collision inputs live in
|
||||||
|
`publish_outputs.go`. Copying, hashing, immutable upload, reuse checks, and
|
||||||
|
remote verification live in `publish_transfer.go`; committed-manifest,
|
||||||
|
immutable-commit, and current-pointer encoding and writes live in
|
||||||
|
`publish_commit.go`. `publish_output_plan.go` owns the deterministic, I/O-free
|
||||||
|
five-outcome model and its secret-free metadata projection.
|
||||||
|
|
||||||
- when publishing or run upload is disabled, completes successfully with no
|
- when publishing or run upload is disabled, completes successfully with no
|
||||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||||
both manifests record success. Enablement and upload policy are fingerprinted,
|
the session manifest records success. Enablement and upload policy are
|
||||||
so changing either automatically makes the prior result non-resumable.
|
fingerprinted, so changing either automatically makes the prior result
|
||||||
|
non-resumable.
|
||||||
- validates prerequisite stage success and object-store availability.
|
- validates prerequisite stage success and object-store availability.
|
||||||
- derives a deterministic run-archive allowlist from the validated run
|
- derives a deterministic run-archive allowlist from the session manifest's
|
||||||
`manifest.json`: declared run-local outputs, logs, generated configs, and the
|
declared current-run outputs, logs, and generated configurations. References
|
||||||
manifest itself. Unlisted workspace files are not archive candidates.
|
outside the current run root, including canonical session files and prior-run
|
||||||
|
diagnostics, are not archive candidates. No distinguished diagnostic file is
|
||||||
|
read or uploaded.
|
||||||
- opens each archive candidate beneath its archive root without following
|
- opens each archive candidate beneath its archive root without following
|
||||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||||
checks a declared checksum when present, then streams the opened descriptor.
|
checks a declared checksum when present, then streams the opened descriptor.
|
||||||
@@ -41,26 +55,34 @@ Exact remote placement and the operator workflow belong in
|
|||||||
- resolves publish output sources through runtime artifact catalog and
|
- resolves publish output sources through runtime artifact catalog and
|
||||||
manifest-aware resolution. Configured Scriptorium outputs are publishable
|
manifest-aware resolution. Configured Scriptorium outputs are publishable
|
||||||
only from validated `current` per-artifact analyze evidence; an incidental
|
only from validated `current` per-artifact analyze evidence; an incidental
|
||||||
canonical file, legacy aggregate output, stale/failed/unselected record, or
|
canonical file, aggregate output, stale/failed/unselected record, or
|
||||||
mismatched path, size, or checksum remains unavailable. This does not change
|
mismatched path, size, or checksum remains unavailable. Built-in and
|
||||||
the explicit compatibility policies owned by built-in, extraction, or
|
extraction sources likewise require their own current manifest evidence.
|
||||||
previous-session sources.
|
|
||||||
- publishes extraction lanes only through explicit configured output rules;
|
- publishes extraction lanes only through explicit configured output rules;
|
||||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||||
- selected artifact filter applies to configured artifact sources only.
|
- selected artifact filter applies to configured artifact sources only.
|
||||||
- locked outputs are skipped intentionally (including required ones).
|
- plans every configured output into exactly one source-sorted outcome:
|
||||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
`published`, `carried_locked`, `locked_unpublished`, `optional_missing`, or
|
||||||
|
`unselected`.
|
||||||
|
- an initially locked selected output carries its exact prior
|
||||||
|
`published_output` declaration when one exists. The declaration may point to
|
||||||
|
an earlier run in the same session and retains its source, key, checksum,
|
||||||
|
size, and generation; the bytes are not copied.
|
||||||
|
- verifies a carried object's exact key, size, and generation through object
|
||||||
|
metadata without downloading or opening the local output source.
|
||||||
|
- an initially locked source with no prior declaration remains absent even
|
||||||
|
when the output rule is required. Optional missing unlocked outputs remain
|
||||||
|
absent; required missing unlocked outputs fail.
|
||||||
- creates one complete immutable source-to-destination mapping before upload;
|
- creates one complete immutable source-to-destination mapping before upload;
|
||||||
- uploads and verifies every declared immutable object and the commit manifest;
|
- uploads and verifies every declared immutable object and the commit manifest;
|
||||||
- updates `current/commit-pointer.json` exactly once, last; and
|
- writes the immutable commit from the combined new and exact carried
|
||||||
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
|
declarations.
|
||||||
- rechecks remote lock state immediately before the pointer update. A newly
|
- rereads every initially unlocked participating source after the immutable
|
||||||
committed lock aborts selection, leaving any uploaded immutable attempt
|
commit. A newly present lock aborts selection, leaving the uploaded immutable
|
||||||
unselected.
|
attempt unselected. Removing an initial lock may conservatively produce one
|
||||||
- reads the mutable remote lock document through a direct limit-plus-one read
|
carry because the initial snapshot controls the plan.
|
||||||
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
|
- updates `current/commit-pointer.json` exactly once, last, and writes no other
|
||||||
with the opened body for conditional updates. Oversized lock documents fail
|
mutable publication control record.
|
||||||
before YAML decoding; published artifact payloads do not use this limit.
|
|
||||||
|
|
||||||
## Metadata Signals
|
## Metadata Signals
|
||||||
|
|
||||||
@@ -68,9 +90,8 @@ Includes counts/lists for:
|
|||||||
- run uploads
|
- run uploads
|
||||||
- published output uploads
|
- published output uploads
|
||||||
- previous uploads
|
- previous uploads
|
||||||
- skipped optional outputs
|
- one `publish_output_outcomes` collection containing all five output states,
|
||||||
- skipped unselected outputs
|
exact declared keys, and key-derived origin runs where applicable
|
||||||
- locked outputs
|
|
||||||
- remote commit and current-pointer key paths
|
- remote commit and current-pointer key paths
|
||||||
- the run identifier selected by the commit
|
- the run identifier selected by the commit
|
||||||
|
|
||||||
@@ -83,35 +104,44 @@ Includes counts/lists for:
|
|||||||
from verified descriptors; symlinks, special files, replacement races, and
|
from verified descriptors; symlinks, special files, replacement races, and
|
||||||
undeclared entries are rejected or ignored before uploads begin.
|
undeclared entries are rejected or ignored before uploads begin.
|
||||||
- run-local diagnostics, including Notarius receipt and stderr files, are
|
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||||
archived only when recorded by the run manifest.
|
archived only when recorded by the session manifest. A run with no declared
|
||||||
- publish locks are not overridden by `--force`; remote locks are revalidated
|
current-run diagnostics legitimately uploads none.
|
||||||
immediately before current-state selection.
|
- independent publish locks are not overridden by `--force`; every initially
|
||||||
- post-commit local cleanup is authorized by the committed publish metadata and
|
unlocked relevant source is reread immediately before current-state
|
||||||
is durably recorded by the application lifecycle before any local deletion.
|
selection.
|
||||||
|
- successful uploaded publication records the exact run ID, remote commit key,
|
||||||
|
and current-pointer key consumed by the later cleanup stage.
|
||||||
|
|
||||||
## Resume Semantics
|
## Resume Semantics
|
||||||
|
|
||||||
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
|
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
|
||||||
normalized source/destination/required output rules, static lock policy, and
|
normalized source/destination/required output rules, and the remote backend,
|
||||||
the remote backend, bucket, region, endpoint, and root-prefix identity. Rule
|
bucket, region, endpoint, and root-prefix identity. Rule ordering is
|
||||||
and lock ordering is canonicalized. Credential environment names,
|
canonicalized. Credential
|
||||||
path-addressing transport mode, local workspace placement, and run identifiers
|
environment names, path-addressing transport mode, local workspace placement,
|
||||||
are excluded. Remote locks remain mutable state and are still revalidated at
|
and run identifiers are excluded. Independent remote locks remain mutable state
|
||||||
the commit boundary; semantic evidence does not replace that safety check.
|
and are read directly by publish at planning and commit boundaries; semantic
|
||||||
|
evidence does not replace those safety checks.
|
||||||
|
|
||||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
The commit boundary and cleanup authorization are normative architecture invariants; see
|
||||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||||
|
|
||||||
|
Publish never deletes local state. The standalone `publish` command selects
|
||||||
|
only this stage; a full run selects notify and cleanup afterward.
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|
||||||
- [Configuration](../config.md#publish-configuration-summary) owns output and
|
- [Configuration](../config.md#publish-configuration-summary) owns output
|
||||||
static-lock fields.
|
rules.
|
||||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||||
physical remote state.
|
physical remote state.
|
||||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||||
helpers.
|
helpers.
|
||||||
- Implementation and tests: `internal/stage/publish.go`,
|
- Implementation and tests: `internal/stage/publish.go`,
|
||||||
|
`internal/stage/publish_sources.go`, `internal/stage/publish_outputs.go`,
|
||||||
|
`internal/stage/publish_output_plan.go`,
|
||||||
|
`internal/stage/publish_transfer.go`, `internal/stage/publish_commit.go`,
|
||||||
`internal/stage/publish_test.go`,
|
`internal/stage/publish_test.go`,
|
||||||
`internal/stage/semantic_contracts_delivery.go`,
|
`internal/stage/semantic_contracts_delivery.go`,
|
||||||
`internal/app/operator_helpers_test.go`, and
|
`internal/app/operator_helpers_test.go`, and
|
||||||
`internal/app/post_publish_cleanup_test.go`
|
`internal/app/cleanup_orchestration_test.go`
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ records that may consume rendered transcripts.
|
|||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||||
- resolves inputs manifest-first, then canonical fallback.
|
- resolves inputs only from verified manifest output evidence.
|
||||||
- writes run-local outputs first, then materializes canonical session outputs.
|
- writes run-local outputs first, then materializes canonical session outputs.
|
||||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||||
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||||
|
|||||||
@@ -14,12 +14,19 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
- discovers prepared audio from manifest inputs or the canonical audio
|
||||||
- derives the transcript identity from the prepared `.flac` filename.
|
directory and derives each speaker identity from its `.flac` filename.
|
||||||
- dispatches WhisperX requests through a bounded worker pool.
|
- hashes the current audio and reconciles it with the versioned per-speaker
|
||||||
- validates each output as JSON.
|
completion collection before starting HTTP work.
|
||||||
- writes run-local outputs then materializes canonical transcript outputs only
|
- reuses only records whose audio identity, observable WhisperX semantics,
|
||||||
after every planned request succeeds.
|
canonical output location, JSON, checksum, and size remain current.
|
||||||
|
- durably checkpoints the reconciled collection before dispatch, then sends
|
||||||
|
only missing or obsolete speakers through the bounded worker group.
|
||||||
|
- validates each successful run-local response, rechecks the audio bytes,
|
||||||
|
materializes the canonical raw transcript, and checkpoints that speaker
|
||||||
|
before accepting another completion.
|
||||||
|
- rebuilds aggregate stage outputs in stable speaker order from the complete
|
||||||
|
current collection, including records produced by earlier invocations.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
@@ -28,21 +35,48 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
|||||||
- output path returned by adapter must match requested output path.
|
- output path returned by adapter must match requested output path.
|
||||||
- an empty adapter result path means the requested path; adapters cannot select
|
- an empty adapter result path means the requested path; adapters cannot select
|
||||||
an alternate destination.
|
an alternate destination.
|
||||||
- each successful output is validated before stage success, and cancellation or
|
- candidate collection construction, manifest save, and installation into live
|
||||||
incomplete dispatch cannot be reported as a successful result.
|
state are serialized. Expensive hashing and HTTP work remain outside that
|
||||||
|
critical section.
|
||||||
|
- a canonical file has no completion authority until its manifest checkpoint
|
||||||
|
succeeds. A save failure preserves the last successfully committed
|
||||||
|
collection for ordinary retry.
|
||||||
|
- each successful output is validated before its checkpoint, and cancellation,
|
||||||
|
incomplete dispatch, or a failed aggregate save cannot be reported as
|
||||||
|
aggregate stage success.
|
||||||
|
- removed audio inputs are omitted from the current collection and aggregate
|
||||||
|
output list; reconciliation does not delete arbitrary transcript files.
|
||||||
|
- merge cannot consume a partial versioned collection: it requires transcribe
|
||||||
|
aggregate success and an exact match between current speaker records and
|
||||||
|
declared aggregate outputs.
|
||||||
|
|
||||||
## Resume Evidence
|
## Resume Evidence
|
||||||
|
|
||||||
Transcribe records a versioned semantic-configuration fingerprint containing
|
Each completed speaker record binds the speaker ID, session-relative audio
|
||||||
the Narratio-visible WhisperX service URL and recognition language. Changes to
|
path, streamed audio checksum and size, the Narratio-visible WhisperX service
|
||||||
either rerun transcription and stale its fixed descendants while leaving
|
URL and recognition language, and the canonical transcript path, checksum,
|
||||||
prepare reusable. Retry count/delay, concurrency, timeout, credentials, and
|
size, and producer run. Changed audio bytes select that speaker for replacement;
|
||||||
diagnostic locations are operational and do not change this evidence.
|
a changed URL or language selects every speaker. Either case stales fixed
|
||||||
|
descendants through the normal runner decision. Retry count/delay, concurrency,
|
||||||
|
timeout, credentials, and diagnostic locations are operational and do not
|
||||||
|
change this evidence.
|
||||||
|
|
||||||
|
An ordinary retry reuses every valid speaker checkpoint even when aggregate
|
||||||
|
transcribe status is failed, running, or stale. A succeeded stage is skipped only
|
||||||
|
when its stored speaker set and aggregate output evidence also match the current
|
||||||
|
collection. Otherwise, execution repairs those records without retranscribing
|
||||||
|
reusable speakers and stales dependents through the normal runner path.
|
||||||
|
`--force` first durably replaces
|
||||||
|
the collection with an empty versioned collection, so a later ordinary retry
|
||||||
|
cannot resurrect pre-force results; speakers checkpointed by that forced
|
||||||
|
attempt remain reusable.
|
||||||
|
|
||||||
WhisperX models or private service configuration not exposed by Narratio's
|
WhisperX models or private service configuration not exposed by Narratio's
|
||||||
adapter contract cannot be fingerprinted; use `--force` after changing them.
|
adapter contract cannot be fingerprinted; use `--force` after changing them.
|
||||||
An existing successful transcribe record without evidence reruns once when
|
An older aggregate-only transcribe record has no trustworthy audio-to-output
|
||||||
selected.
|
binding. When transcribe is next selected, every current speaker is therefore
|
||||||
|
generated once and recorded in the versioned collection; later invocations can
|
||||||
|
reuse it normally.
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|
||||||
@@ -51,6 +85,9 @@ selected.
|
|||||||
- [Configuration](../config.md#pipeline) owns concurrency and other
|
- [Configuration](../config.md#pipeline) owns concurrency and other
|
||||||
operator-selected values.
|
operator-selected values.
|
||||||
- Implementation and tests: `internal/stage/transcribe.go`,
|
- Implementation and tests: `internal/stage/transcribe.go`,
|
||||||
|
`internal/stage/transcribe_reconciliation.go`,
|
||||||
|
`internal/stage/transcribe_execution_state.go`,
|
||||||
|
`internal/stage/transcribe_resume.go`,
|
||||||
`internal/stage/transcribe_test.go`,
|
`internal/stage/transcribe_test.go`,
|
||||||
`internal/stage/semantic_contracts_initial.go`, and
|
`internal/stage/transcribe_reconciliation_test.go`, and
|
||||||
`internal/stage/semantic_contracts_initial_test.go`
|
`internal/stage/transcribe_checkpoint_test.go`
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ operator-selected storage fields and credential mechanisms belong in
|
|||||||
- `Download(ctx, key, localPath)`
|
- `Download(ctx, key, localPath)`
|
||||||
- `Upload(ctx, localPath, key, opts)`
|
- `Upload(ctx, localPath, key, opts)`
|
||||||
- `UploadConditional(ctx, source, key, opts, condition)`
|
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||||
|
- `Delete(ctx, key)` removes one object and succeeds when it is already absent
|
||||||
- `Exists(ctx, key)`
|
- `Exists(ctx, key)`
|
||||||
|
|
||||||
Key invariant:
|
Key invariant:
|
||||||
@@ -47,6 +48,8 @@ not own discovery, defaults, or configuration validation.
|
|||||||
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||||
match or absence precondition directly to the provider request and reports a
|
match or absence precondition directly to the provider request and reports a
|
||||||
failed precondition without performing a local check-then-write replacement.
|
failed precondition without performing a local check-then-write replacement.
|
||||||
|
- `Delete` maps one normalized key to the provider delete-object operation and
|
||||||
|
treats an absent object as success.
|
||||||
- `Exists` maps not-found responses to `false`.
|
- `Exists` maps not-found responses to `false`.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
@@ -55,6 +58,11 @@ not own discovery, defaults, or configuration validation.
|
|||||||
- bounded reads never retain more than the caller's limit plus one byte and do
|
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||||
not replace owner-specific size policy.
|
not replace owner-specific size policy.
|
||||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||||
|
- A caller that installs remote content into a canonical workspace path first
|
||||||
|
writes a sibling temporary file, validates the complete object against its
|
||||||
|
owner-specific contract, and only then replaces the destination. Temporary
|
||||||
|
files used solely for remote comparisons are removed by their callers and do
|
||||||
|
not represent installed workspace state.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
|||||||
consume those helpers instead of rebuilding relative paths.
|
consume those helpers instead of rebuilding relative paths.
|
||||||
|
|
||||||
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||||
cleanup roots and their descendants through no-follow directory handles before
|
explicit cleanup roots and removes only normalized proper descendants through
|
||||||
removing them.
|
the resulting root handle.
|
||||||
|
|
||||||
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||||
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||||
@@ -33,15 +33,16 @@ previous-session cache paths remain session-durable and are never redirected
|
|||||||
into run-local output space.
|
into run-local output space.
|
||||||
|
|
||||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
validates and renames the external bundle to the unique immutable Notarius
|
||||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
||||||
copies only regular files and directories to a same-filesystem temporary
|
|
||||||
sibling. Source traversal uses confined directory handles and identity checks
|
|
||||||
so replacing an inspected root, directory, or file is rejected rather than
|
|
||||||
followed. The completed tree is atomically renamed without replacing an
|
|
||||||
existing destination. Exact physical paths belong in
|
|
||||||
[Operations](../operations.md#extraction-workflow).
|
[Operations](../operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
Prepared references delivered to Notarius use session-durable paths owned by
|
||||||
|
`internal/artifacts` beneath
|
||||||
|
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
|
||||||
|
content-addressed snapshots are outside run-local layout, survive ordinary
|
||||||
|
published-run cleanup, and are excluded from publish and restore enumeration.
|
||||||
|
|
||||||
## Locking
|
## Locking
|
||||||
|
|
||||||
`artifacts.LocalStore` enforces the single-writer session lock via an
|
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||||
@@ -50,21 +51,25 @@ file retains owner metadata after release or process death; its existence is
|
|||||||
not evidence that a lock is active. Command and restore flows wait for this
|
not evidence that a lock is active. Command and restore flows wait for this
|
||||||
lock only while their context remains active, and report a release failure.
|
lock only while their context remains active, and report a release failure.
|
||||||
|
|
||||||
|
The runner assigns every invocation a run-workspace identity but creates the
|
||||||
|
physical directory only before an executing stage that requires it. Fully
|
||||||
|
reused invocations and cleanup-only invocations therefore do not accumulate
|
||||||
|
empty run directories.
|
||||||
|
|
||||||
## Cleanup Semantics
|
## Cleanup Semantics
|
||||||
|
|
||||||
Automatic post-publish cleanup:
|
The canonical cleanup stage:
|
||||||
|
|
||||||
- is created only after a successful publish commit with complete publish
|
- runs after notify only when selected by the invocation range;
|
||||||
metadata, then is persisted before any deletion;
|
- requires a succeeded publish record with `uploaded=true`, the published run
|
||||||
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
|
ID, remote commit key, and current commit-pointer key;
|
||||||
key in publish metadata;
|
|
||||||
- consumes the resolved cleanup policy described in
|
- consumes the resolved cleanup policy described in
|
||||||
[Configuration](../config.md);
|
[Configuration](../config.md);
|
||||||
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
|
- refuses unsafe deletes (root delete, traversal, and out-of-root targets);
|
||||||
ancestors or entries);
|
- treats missing targets as successful, idempotent deletion; and
|
||||||
- retries any recorded incomplete target on later invocations even when no
|
- uses its ordinary failed/succeeded manifest stage record for retry. A later
|
||||||
publish work is selected. Missing targets are a successful, idempotent
|
full run can skip reusable upstream stages and retry cleanup without
|
||||||
cleanup result only after the completion evidence is saved.
|
republishing.
|
||||||
|
|
||||||
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||||
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||||
@@ -75,6 +80,8 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
|||||||
- campaign-aware session root is mandatory.
|
- campaign-aware session root is mandatory.
|
||||||
- manifest-driven stage state is durable across runs.
|
- manifest-driven stage state is durable across runs.
|
||||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||||
|
- restore installation is confined to validated relative paths beneath the
|
||||||
|
selected session root.
|
||||||
- ordinary workspace paths retain group-writable directory and file modes across
|
- ordinary workspace paths retain group-writable directory and file modes across
|
||||||
nested creation, replacement, and Notarius promotion.
|
nested creation, replacement, and Notarius promotion.
|
||||||
|
|
||||||
@@ -83,12 +90,14 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
|||||||
- Path model and local store: `internal/artifacts/paths.go`,
|
- Path model and local store: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/local.go`
|
`internal/artifacts/local.go`
|
||||||
- Run-local materialization: `internal/stage/run_local.go`
|
- Run-local materialization: `internal/stage/run_local.go`
|
||||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
- Immutable bundle installation: `internal/stage/extract.go`
|
||||||
- Workspace modes: `internal/fileops/modes.go`
|
- Workspace modes: `internal/fileops/modes.go`
|
||||||
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||||
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
|
`internal/stage/cleanup.go`, `internal/stage/cleanup_plan.go`, and
|
||||||
|
`internal/app/cleanup_targets.go`
|
||||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||||
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
|
`internal/fileops/modes_posix_test.go`,
|
||||||
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||||
`internal/app/post_publish_cleanup_test.go`
|
`internal/stage/cleanup_test.go`, and
|
||||||
|
`internal/app/cleanup_orchestration_test.go`
|
||||||
|
|||||||
@@ -76,12 +76,15 @@ value from the last value persisted in the session manifest, which helps
|
|||||||
diagnose profile switches without changing resume authority.
|
diagnose profile switches without changing resume authority.
|
||||||
|
|
||||||
Before switching an operational profile, compare its effective meaning with the
|
Before switching an operational profile, compare its effective meaning with the
|
||||||
current selection through `narratio config diff <left-profile> <right-profile>`.
|
production base through `narratio config diff default <profile>`.
|
||||||
The command is read-only and succeeds whether it finds differences or not. Its
|
The command is read-only and succeeds whether it finds differences or not. Its
|
||||||
sorted records describe defaulted, expanded concrete configuration—not source
|
sorted records describe defaulted, expanded concrete configuration—not source
|
||||||
file layout—so it can be used to review model, artifact, and publish changes
|
file layout—so it can be used to review model, artifact, and publish changes
|
||||||
without creating a session or run. Select the same campaign explicitly when
|
without creating a session or run. Use `narratio config artifacts` to review the
|
||||||
profiles could resolve different campaign paths; see the [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) for syntax and record format.
|
concrete artifact identities, definitions, outputs, and dependencies. Select the
|
||||||
|
same campaign explicitly when profiles could resolve different campaign paths;
|
||||||
|
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
|
for syntax and record format.
|
||||||
|
|
||||||
## Stage Execution and Continuation Behavior
|
## Stage Execution and Continuation Behavior
|
||||||
|
|
||||||
@@ -98,12 +101,12 @@ Canonical stage order:
|
|||||||
9. `analyze`
|
9. `analyze`
|
||||||
10. `publish`
|
10. `publish`
|
||||||
11. `notify`
|
11. `notify`
|
||||||
|
12. `cleanup`
|
||||||
|
|
||||||
Execution rules:
|
Execution rules:
|
||||||
|
|
||||||
- succeeded stages are skipped unless `--force` is set; stages with semantic
|
- succeeded stages are skipped unless `--force` is set; stages with semantic
|
||||||
configuration contracts additionally require matching versioned evidence,
|
configuration contracts additionally require matching versioned evidence;
|
||||||
and missing legacy evidence causes a safe one-time rerun;
|
|
||||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- forcing a stage marks succeeded transitive dependents as `stale` before the
|
- forcing a stage marks succeeded transitive dependents as `stale` before the
|
||||||
replacement runs; render and extract are independent siblings; and
|
replacement runs; render and extract are independent siblings; and
|
||||||
@@ -113,8 +116,8 @@ Execution rules:
|
|||||||
perpetually rerun dependent work.
|
perpetually rerun dependent work.
|
||||||
|
|
||||||
Every aggregate stage except analyze currently provides semantic-configuration
|
Every aggregate stage except analyze currently provides semantic-configuration
|
||||||
evidence; analyze retains its more precise per-artifact fingerprints and
|
evidence; transcribe additionally reconciles per-speaker completion evidence,
|
||||||
validator.
|
while analyze retains its more precise per-artifact fingerprints and validator.
|
||||||
Prepare additionally validates current stable/local/S3 source identity and the
|
Prepare additionally validates current stable/local/S3 source identity and the
|
||||||
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
||||||
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
||||||
@@ -155,7 +158,7 @@ narratio run 2026-04-04 --from extract --through analyze --force
|
|||||||
```
|
```
|
||||||
|
|
||||||
Omitting `--from` selects from `prepare`; omitting `--through` selects through
|
Omitting `--from` selects from `prepare`; omitting `--through` selects through
|
||||||
`notify`. Force applies only within the selected range. Repeating `--from`,
|
`cleanup`. Force applies only within the selected range. Repeating `--from`,
|
||||||
`--through`, or `--force` is rejected instead of resolving by argument order.
|
`--through`, or `--force` is rejected instead of resolving by argument order.
|
||||||
The plan command uses the same selection contract and prints only the selected
|
The plan command uses the same selection contract and prints only the selected
|
||||||
range. Planning is read-only: it clones the loaded manifest, models selected
|
range. Planning is read-only: it clones the loaded manifest, models selected
|
||||||
@@ -169,11 +172,11 @@ selected artifact evidence is still semantically current.
|
|||||||
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
||||||
prefix stage must already have a session-manifest status of `succeeded` or
|
prefix stage must already have a session-manifest status of `succeeded` or
|
||||||
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
||||||
or interrupted prerequisite without creating a run record or changing session
|
or interrupted prerequisite without creating a run-local directory or changing
|
||||||
state. Widen `--from` to include that stage, or recover it explicitly before
|
session state. Widen `--from` to include that stage, or recover it explicitly
|
||||||
retrying. Excluded prefix stages are not resume-validated or repaired as part
|
before retrying. Excluded prefix stages are not resume-validated or repaired as
|
||||||
of the bounded invocation; selected stages still reject missing, unsafe, or
|
part of the bounded invocation; selected stages still reject missing, unsafe,
|
||||||
manifest-inconsistent inputs at their owning boundary.
|
or manifest-inconsistent inputs at their owning boundary.
|
||||||
|
|
||||||
Stages after `--through` are not prerequisites and are never scheduled by the
|
Stages after `--through` are not prerequisites and are never scheduled by the
|
||||||
bounded invocation. A selected forced stage can mark one of those succeeded
|
bounded invocation. A selected forced stage can mark one of those succeeded
|
||||||
@@ -184,7 +187,68 @@ shared session lifecycle. In particular, render does not require Notarius or
|
|||||||
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
||||||
the transcription, Seriatim, Audita, or Notarius adapters.
|
the transcription, Seriatim, Audita, or Notarius adapters.
|
||||||
|
|
||||||
For the common post-transcript development loop, use:
|
### Transcription continuation and replacement
|
||||||
|
|
||||||
|
Transcribe checkpoints each validated speaker independently. If a later
|
||||||
|
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
|
||||||
|
save fails, the aggregate stage is not successful and downstream merge cannot
|
||||||
|
consume the partial set. The last successfully saved speaker records and their
|
||||||
|
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
|
||||||
|
|
||||||
|
After correcting the underlying problem, retry normally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio hashes the current prepared audio, validates recorded transcript JSON,
|
||||||
|
checksum, size, and location, and calls WhisperX only for missing or obsolete
|
||||||
|
speakers. A changed audio file replaces only its speaker; a changed WhisperX
|
||||||
|
URL or language makes every current speaker non-reusable. Retry, timeout, and
|
||||||
|
concurrency tuning do not invalidate completed speakers.
|
||||||
|
|
||||||
|
Use force only when every current transcript must be replaced, including after
|
||||||
|
a WhisperX model or private server-side configuration changes behind the same
|
||||||
|
URL and language:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Force durably clears the reusable collection before sending requests. If that
|
||||||
|
forced attempt fails, a later ordinary retry reuses only replacement speakers
|
||||||
|
checkpointed after the reset; it does not revive pre-force results. Do not edit
|
||||||
|
speaker checksums or copy older JSON into the canonical directory to influence
|
||||||
|
reuse.
|
||||||
|
|
||||||
|
The first current Narratio invocation that selects transcribe for an older
|
||||||
|
aggregate-only manifest retranscribes every current speaker once because the
|
||||||
|
old record cannot prove which audio bytes produced each file. Subsequent runs
|
||||||
|
reuse the new versioned evidence normally. The overall session-manifest schema
|
||||||
|
identifier is unchanged, but older strict-decoding Narratio binaries may reject
|
||||||
|
a manifest after current Narratio writes the additive speaker fields; retain a
|
||||||
|
backup or complete the upgrade before relying on downgrade.
|
||||||
|
|
||||||
|
Restoring a manifest does not implicitly publish or restore every raw speaker
|
||||||
|
file. If a recorded canonical transcript is absent locally, ordinary
|
||||||
|
reconciliation schedules that speaker again. Removing run directories does not
|
||||||
|
remove canonical checkpointed transcripts and does not by itself invalidate
|
||||||
|
them.
|
||||||
|
|
||||||
|
## Artifact Development Workflows
|
||||||
|
|
||||||
|
Choose the narrowest workflow that matches the intended replacement:
|
||||||
|
|
||||||
|
| Intent | Command | Preserved and rebuilt state |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
|
||||||
|
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
|
||||||
|
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
|
||||||
|
|
||||||
|
All three stop before publication. Inspect the results and run
|
||||||
|
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
|
||||||
|
|
||||||
|
For the extraction-plus-analysis development loop, use:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio regenerate-artifacts 2026-04-04
|
narratio regenerate-artifacts 2026-04-04
|
||||||
@@ -195,8 +259,8 @@ This command is a transparent expansion to a forced bounded `run` from
|
|||||||
`extract` through `analyze`. Extraction always rebuilds its complete configured
|
`extract` through `analyze`. Extraction always rebuilds its complete configured
|
||||||
bundle. Analysis rebuilds the selected targets and their required analysis
|
bundle. Analysis rebuilds the selected targets and their required analysis
|
||||||
prerequisites, or uses the normal default selection when no artifact names are
|
prerequisites, or uses the normal default selection when no artifact names are
|
||||||
given. The command does not run publish or notify; delivery remains a separate
|
given. The command does not run publish, notify, or cleanup; delivery and
|
||||||
operator action.
|
retention remain separate operator actions.
|
||||||
|
|
||||||
Inspect current artifact evidence, then publish explicitly when the regenerated
|
Inspect current artifact evidence, then publish explicitly when the regenerated
|
||||||
set is ready:
|
set is ready:
|
||||||
@@ -206,11 +270,41 @@ narratio session artifacts 2026-04-04
|
|||||||
narratio publish 2026-04-04
|
narratio publish 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
If planning or execution reports stale, missing, failed, legacy, or tampered
|
If planning or execution reports stale, missing, failed, or tampered
|
||||||
analysis evidence, regenerate the affected target instead of copying an older
|
analysis evidence, regenerate the affected target instead of copying an older
|
||||||
canonical file into place or editing the manifest. See
|
canonical file into place or editing the manifest. See
|
||||||
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
|
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
|
||||||
|
|
||||||
|
Analyze checkpoints each completed configured artifact before starting later
|
||||||
|
artifact work. After a partial failure or interruption, retry normally: valid
|
||||||
|
earlier checkpoints are reused and execution resumes with failed, stale, or
|
||||||
|
missing work in dependency order. `--force` is unnecessary unless the operator
|
||||||
|
intends to replace otherwise-current artifacts.
|
||||||
|
|
||||||
|
### Developing With A Notarius Configuration Profile
|
||||||
|
|
||||||
|
A Narratio development overlay can explicitly select a Notarius overlay without
|
||||||
|
requiring the names to match. For example, `profiles/testing.yml` may contain:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: downstream-economy
|
||||||
|
```
|
||||||
|
|
||||||
|
Selecting Narratio's `testing` profile then supplies `downstream-economy` only
|
||||||
|
when extraction executes. Switching that explicit selector makes a recorded
|
||||||
|
extraction non-current and reruns it when `extract` is in the selected stage
|
||||||
|
range; the usual dependent invalidation then applies. Selecting a profile does
|
||||||
|
not create a separate workspace or retain a separate current extraction result.
|
||||||
|
|
||||||
|
Narratio cannot observe edits inside Notarius configuration behind an unchanged
|
||||||
|
selector. Force extraction after such an edit when a replacement is intended.
|
||||||
|
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
|
||||||
|
use `analyze` when the existing extraction bundle should be preserved. Field
|
||||||
|
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
|
||||||
|
and the exact downstream invocation belongs in the
|
||||||
|
[Notarius integration contract](./integrations/notarius.md#invocation).
|
||||||
|
|
||||||
## Artifact Selection
|
## Artifact Selection
|
||||||
|
|
||||||
For a configured artifact family, selecting its family key expands to every
|
For a configured artifact family, selecting its family key expands to every
|
||||||
@@ -251,9 +345,14 @@ are not selectable or published implicitly.
|
|||||||
Configured Notarius references resolve only from the current manifest-backed
|
Configured Notarius references resolve only from the current manifest-backed
|
||||||
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
||||||
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
||||||
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified copies
|
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
|
||||||
under `runs/<run_id>/extract/references/` so a concurrent refresh of canonical
|
content-addressed copies under
|
||||||
prepared files cannot change the bytes consumed by an in-flight invocation.
|
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
|
||||||
|
SHA-256 digest identifies the reference bytes, so retries and later invocations
|
||||||
|
reuse the same path while changed bytes select a new path. Older snapshots are
|
||||||
|
retained and are not current artifact evidence. A concurrent refresh of
|
||||||
|
canonical prepared files therefore cannot change the bytes consumed by an
|
||||||
|
in-flight invocation.
|
||||||
For a canonical party, preparation retains the validated authored party bytes
|
For a canonical party, preparation retains the validated authored party bytes
|
||||||
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
||||||
The manifest records their checksums separately, with the players input marked
|
The manifest records their checksums separately, with the players input marked
|
||||||
@@ -280,15 +379,35 @@ narratio run-stage prepare 2026-04-04 --force
|
|||||||
|
|
||||||
Starting a replacement clears the previous extraction payload from the current
|
Starting a replacement clears the previous extraction payload from the current
|
||||||
session-stage record. If that replacement fails or self-skips, the current
|
session-stage record. If that replacement fails or self-skips, the current
|
||||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
record does not fall back to the earlier outputs. The earlier immutable bundle
|
||||||
immutable bundle remain available for inspection, but downstream resolution
|
remains available for inspection, but downstream resolution requires a new
|
||||||
requires a new current successful extraction record.
|
current successful extraction record.
|
||||||
|
|
||||||
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
|
Notarius extraction validates its run-local bundle and moves it once to the
|
||||||
and other operating systems, extraction fails before copying the bundle into a
|
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
no duplicate promotion tree is created.
|
||||||
directory primitive there. This is an extraction limitation, not a broader
|
|
||||||
platform-support guarantee for every Narratio workflow.
|
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
|
||||||
|
inside an executed Notarius invocation. Ordinary extraction forwards their
|
||||||
|
configured values. Forced extraction always disables checkpoint loading and
|
||||||
|
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
|
||||||
|
Refresh regenerates and replaces only the active input's accepted chunk plan;
|
||||||
|
it is not cache deletion.
|
||||||
|
|
||||||
|
Notarius checkpoint recording must be enabled in the deployed Notarius
|
||||||
|
configuration. Narratio neither creates a Notarius pipeline/configuration nor
|
||||||
|
reads it to infer checkpoint availability. Configure explicit persistent
|
||||||
|
Notarius chunk-plan and checkpoint roots when reuse must survive separate
|
||||||
|
Narratio subprocesses. A changed reference, transcript, selected profile,
|
||||||
|
profile content, endpoint, or other Notarius checkpoint identity input can
|
||||||
|
produce a legitimate cold miss.
|
||||||
|
|
||||||
|
There are two independent resume boundaries: Narratio may skip a completed
|
||||||
|
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
|
||||||
|
reuse compatible accepted work only after Narratio has decided extraction must
|
||||||
|
execute. Deleting Narratio's local reference snapshots does not invalidate a
|
||||||
|
complete durable extraction; missing snapshots are reconstructed only when a
|
||||||
|
later extraction actually runs.
|
||||||
|
|
||||||
## External Command Lifecycle
|
## External Command Lifecycle
|
||||||
|
|
||||||
@@ -311,12 +430,15 @@ Run-local diagnostics are:
|
|||||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||||
|
|
||||||
The run-record upload is an allowlist derived from the validated run manifest,
|
The run upload is an allowlist derived from the session manifest's declared
|
||||||
not a workspace scan. Each declared source is opened without following
|
current-run outputs, logs, and generated configurations, not a workspace scan.
|
||||||
symlinked ancestors or the leaf, verified as a regular file, and streamed from
|
References to canonical session files and prior-run paths are excluded. Each
|
||||||
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
|
declared source is opened without following symlinked ancestors or the leaf,
|
||||||
The durable bundle is never scanned for implicit publication; only lanes named
|
verified as a regular file, and streamed from that verified descriptor.
|
||||||
by explicit `pipeline.publish.outputs` rules are uploaded.
|
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
||||||
|
current-run diagnostics uploads no run-local files. No distinguished diagnostic
|
||||||
|
file is uploaded. The durable bundle is never scanned for implicit publication;
|
||||||
|
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||||
|
|
||||||
To intentionally replace the current extraction result, run:
|
To intentionally replace the current extraction result, run:
|
||||||
|
|
||||||
@@ -327,22 +449,23 @@ narratio run-stage extract 2026-04-04 --force
|
|||||||
Narratio automatically reruns extraction when its recorded invocation contract,
|
Narratio automatically reruns extraction when its recorded invocation contract,
|
||||||
prepared Narratio reference identities, or durable output validation changes.
|
prepared Narratio reference identities, or durable output validation changes.
|
||||||
The semantic portion covers Notarius enablement, pipeline identity, declared
|
The semantic portion covers Notarius enablement, pipeline identity, declared
|
||||||
reference mapping, and output contracts. Executable, timeout, working directory,
|
reference mapping, output contracts, and the explicit Notarius configuration
|
||||||
and private config-file paths are operational and do not invalidate a current
|
profile selector. Executable, timeout, working directory, and private
|
||||||
result.
|
config-file paths are operational and do not invalidate a current result.
|
||||||
It cannot fingerprint configuration files, profiles, prompts, modules, or
|
It cannot fingerprint configuration or profile contents, prompts, modules, or
|
||||||
other references loaded transitively by Notarius itself. Force extraction after
|
other references loaded transitively by Notarius itself. Force extraction after
|
||||||
changing any of those inputs, even when the top-level Narratio and Notarius
|
changing any of those inputs behind an unchanged selector, even when the
|
||||||
config paths remain the same. A forced extract
|
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||||
marks successful downstream stages stale. Ordinary extraction failures or
|
marks successful downstream stages stale. Ordinary extraction failures or
|
||||||
outcome changes also stale affected downstream stages, while an identical
|
outcome changes also stale affected downstream stages, while an identical
|
||||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||||
|
|
||||||
Publish reuse additionally tracks enabled/run-upload behavior, normalized
|
Publish reuse additionally tracks enabled/run-upload behavior, normalized
|
||||||
output rules, static locks, and remote backend/bucket/region/endpoint/root
|
output rules, and remote backend/bucket/region/endpoint/root identity.
|
||||||
identity. Credential environment names, local workspace placement, and run IDs
|
Credential environment names, local workspace placement, and run IDs are
|
||||||
are excluded. Regardless of semantic reuse evidence, executing publish still
|
excluded. Regardless of semantic reuse evidence, executing publish reads
|
||||||
revalidates mutable remote locks immediately before commit selection.
|
independent remote locks directly and rechecks initially unlocked sources
|
||||||
|
before commit selection.
|
||||||
|
|
||||||
## Publish Workflow
|
## Publish Workflow
|
||||||
|
|
||||||
@@ -358,6 +481,9 @@ Equivalent:
|
|||||||
narratio run-stage publish 2026-04-04 --force
|
narratio run-stage publish 2026-04-04 --force
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This standalone command stops after publish and does not invoke cleanup. A
|
||||||
|
default full `run` continues through notify and cleanup.
|
||||||
|
|
||||||
Publish commit model:
|
Publish commit model:
|
||||||
|
|
||||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||||
@@ -381,31 +507,44 @@ commit by version, checksum, size, and storage generation; committed artifacts
|
|||||||
are also checksum- and generation-bound. Readers accept this contract now and
|
are also checksum- and generation-bound. Readers accept this contract now and
|
||||||
strictly reject mismatched or unknown data.
|
strictly reject mismatched or unknown data.
|
||||||
|
|
||||||
Legacy reads are limited to a coherent `current/manifest.json` and
|
A selected commit may carry a `published_output` declaration from an earlier
|
||||||
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
|
run in the same campaign/session. Restore, previous-session caching, status,
|
||||||
write that pair and remote commit state does not carry local
|
and artifact inspection use that declaration's exact immutable key and stable
|
||||||
`current_pointer_written` metadata.
|
source; they do not reconstruct a current-run key or scan the session prefix.
|
||||||
|
Other artifact types remain confined to the selected commit's own run.
|
||||||
|
|
||||||
|
When the pointer is absent, remote state is not published; republish the
|
||||||
|
session to establish a committed snapshot. Remote commit state does not carry
|
||||||
|
local `current_pointer_written` metadata.
|
||||||
|
|
||||||
## Publish Locks
|
## Publish Locks
|
||||||
|
|
||||||
Lock sources:
|
Each mutable lock is an independent strict JSON object at
|
||||||
|
`{session_prefix}/locks/{sha256(source)}.lock`. The lowercase digest is computed
|
||||||
|
from the exact canonical source. Listing validates every object in that
|
||||||
|
reserved prefix and reports locks sorted by source.
|
||||||
|
|
||||||
- static locks in `pipeline.publish.locks`
|
Upgrades do not read, migrate, or delete the retired `{session_prefix}/locks.yml`
|
||||||
- mutable remote locks in `{session_prefix}/locks.yml`
|
object. It has no effect on publication and may be removed manually after
|
||||||
|
verifying lock commands and publication behavior with the upgraded Narratio
|
||||||
|
version. No migration command is required.
|
||||||
|
|
||||||
Effective lock rules:
|
Operational rules:
|
||||||
|
|
||||||
- static and remote locks are merged;
|
- add uses conditional creation and reports an existing lock;
|
||||||
- static locks win on source collisions;
|
- add with `--force` observes the current generation and attempts one
|
||||||
- locked outputs are intentional skips;
|
generation-matched replacement. A concurrent change is reported for an
|
||||||
- lock add/remove commands mutate only remote lock state through generation-bound
|
explicit operator retry;
|
||||||
conditional writes. A command retries a bounded number of concurrent
|
- remove is idempotent and reports whether a lock was present;
|
||||||
conflicts while its invocation context remains active, so it never replaces a
|
- the initial publish lock snapshot controls output planning. A locked source
|
||||||
different lock-document generation; and
|
carries its exact declaration from the prior current commit when available;
|
||||||
- a publish re-reads remote locks immediately before it writes the current
|
a never-published locked source remains absent even when required;
|
||||||
commit pointer. A lock committed before that recheck prevents selecting the
|
- carried bytes are not copied. Publish verifies the prior object's exact key,
|
||||||
new snapshot, even though its already-uploaded immutable objects may remain
|
size, and generation without downloading it; and
|
||||||
available for a later retry.
|
- after immutable uploads and the immutable commit, publish re-reads every
|
||||||
|
initially unlocked participating source. A newly added lock prevents the
|
||||||
|
current pointer from advancing, although the immutable attempt may remain.
|
||||||
|
A removed initial lock may conservatively cause one carry.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
@@ -439,13 +578,22 @@ Default restore scope:
|
|||||||
|
|
||||||
- the committed session manifest and the committed transcript/artifact objects
|
- the committed session manifest and the committed transcript/artifact objects
|
||||||
declared by the selected remote commit
|
declared by the selected remote commit
|
||||||
- `previous/**` when needed by configured previous-session artifact inputs
|
- `previous/**` when needed by configured previous-session artifact inputs;
|
||||||
|
each is selected from its exact committed `published_output` declaration
|
||||||
|
|
||||||
Optional:
|
Optional:
|
||||||
|
|
||||||
- `--include-audio` to include `audio/**`
|
- `--include-audio` to include `audio/**`
|
||||||
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||||
directories or other non-regular local targets
|
directories, symlinks, or other non-regular local targets
|
||||||
|
|
||||||
|
Restore treats a local leaf symlink or any symlinked ancestor beneath the
|
||||||
|
selected session root as a conflict, even when the linked content matches the
|
||||||
|
selected remote object. `--force` does not override that conflict.
|
||||||
|
|
||||||
|
Before each install, restore validates the destination again beneath the
|
||||||
|
selected session root. A target changed into a symlink, directory, or other
|
||||||
|
non-regular entry is not overwritten.
|
||||||
|
|
||||||
Restore writes an execution report at `reports/restore-latest.json`.
|
Restore writes an execution report at `reports/restore-latest.json`.
|
||||||
|
|
||||||
@@ -473,10 +621,24 @@ Durable session paths:
|
|||||||
- `logs/**`
|
- `logs/**`
|
||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
- `cache/notarius/references/<sha256>/<prepared-basename>`
|
||||||
|
|
||||||
|
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||||
|
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||||
|
add missing markers. Move it aside and regenerate it, restore a verified
|
||||||
|
current committed snapshot, or republish healthy regenerated state as the
|
||||||
|
relevant workflow requires.
|
||||||
|
|
||||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||||
|
Stable reference snapshots live in the session cache path above. Ordinary
|
||||||
|
published-run cleanup removes its selected run-local target but retains this
|
||||||
|
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
|
||||||
|
or digest directory manually only while no command is active for the session;
|
||||||
|
regular missing or corrupt content is reconstructed on the next executed
|
||||||
|
extraction, while symlinks, directories, and other unsafe conflicts must be
|
||||||
|
investigated and removed deliberately.
|
||||||
|
|
||||||
Run-local layout:
|
Run-local layout:
|
||||||
|
|
||||||
@@ -498,6 +660,8 @@ Cache layout (durable S3 audio cache):
|
|||||||
Each cached audio file has an adjacent managed identity record. It binds the
|
Each cached audio file has an adjacent managed identity record. It binds the
|
||||||
file to its remote object version and verified digest; deleting or altering the
|
file to its remote object version and verified digest; deleting or altering the
|
||||||
record simply causes Narratio to download and verify the object again.
|
record simply causes Narratio to download and verify the object again.
|
||||||
|
Remote audio and restored files are fully transferred into sibling temporary
|
||||||
|
files and validated before they replace their canonical workspace paths.
|
||||||
|
|
||||||
### Workspace Permissions
|
### Workspace Permissions
|
||||||
|
|
||||||
@@ -509,18 +673,28 @@ converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
|||||||
caller umask does not remove group write or setgid. It does not change file or
|
caller umask does not remove group write or setgid. It does not change file or
|
||||||
directory ownership: the configured workspace's existing group is inherited.
|
directory ownership: the configured workspace's existing group is inherited.
|
||||||
|
|
||||||
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
|
||||||
workspace, spool, and cache locations with an ACL that grants the collaborating
|
|
||||||
group read/write access, and configure credential locations with an ACL limited
|
|
||||||
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
|
||||||
Windows access control.
|
|
||||||
|
|
||||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||||
shared workspace or in a separately restricted credential location; ordinary
|
shared workspace or in a separately restricted credential location; ordinary
|
||||||
workspace group access must never be treated as authorization to read keys.
|
workspace group access must never be treated as authorization to read keys.
|
||||||
On POSIX, provision a credential directory as `0700` and credential files as
|
Provision a credential directory as `0700` and credential files as `0600`;
|
||||||
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
these permission modes are an operator requirement and are not enforced by
|
||||||
On Windows, restrict the directory and files with ACLs to the credential owner.
|
ConfigKit.
|
||||||
|
|
||||||
|
Every pipeline-loading command bootstraps credentials once, before the first
|
||||||
|
configuration load and before any adapter is constructed. `--secrets-dir`
|
||||||
|
selects an explicit directory (relative paths use caller CWD). Otherwise a
|
||||||
|
present `secrets/` beside the resolved root pipeline is used, while an absent
|
||||||
|
conventional directory is skipped. Explicit missing/empty selections and
|
||||||
|
invalid existing directories fail the command. ConfigKit follows a symlink used
|
||||||
|
as the selected directory, but immediate entries must be regular non-symlink
|
||||||
|
files. Existing environment variables—including empty values—always win.
|
||||||
|
|
||||||
|
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
|
||||||
|
environment references. When both are absent, the AWS SDK retains its default
|
||||||
|
credential provider chain. When both are configured, Narratio supplies their
|
||||||
|
load-time resolved values through an explicit static provider. Audita's optional
|
||||||
|
`llm_api_key` is likewise captured at pipeline load and supplied only through
|
||||||
|
the child process's `AUDITA_LLM_API_KEY` override.
|
||||||
|
|
||||||
External adapter results are individually bounded before Narratio validates or
|
External adapter results are individually bounded before Narratio validates or
|
||||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||||
@@ -553,17 +727,29 @@ Rules:
|
|||||||
- `clean` deletes work/spool session state;
|
- `clean` deletes work/spool session state;
|
||||||
- cache is preserved unless `--clear-cache` is set;
|
- cache is preserved unless `--clear-cache` is set;
|
||||||
- each deletion is confined beneath its configured workspace, spool, or cache
|
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||||
root and refuses symlinked paths;
|
root; root, traversal, and out-of-root targets are refused;
|
||||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
- the final cleanup stage is gated by successful publish evidence plus:
|
||||||
- `pipeline.spool.delete_audio_after_publish=true`
|
- `pipeline.spool.delete_audio_after_publish=true`
|
||||||
- `pipeline.workspace.cleanup_after_publish=true`
|
- `pipeline.workspace.cleanup_after_publish=true`
|
||||||
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
|
- the manifest publish record has `uploaded=true`, the published run ID, the
|
||||||
reports incomplete, the remote committed snapshot remains current; rerun
|
exact remote commit key, and the current commit-pointer key.
|
||||||
publish to retry only the outstanding confined local cleanup.
|
|
||||||
|
|
||||||
Post-publish cleanup is evaluated only when `publish` actually executes in the
|
Cleanup derives only the configured run-scoped spool and workspace targets,
|
||||||
current invocation. A bounded range that excludes publish does not replay a
|
uses root-confined deletion, and does not read or mutate object storage. A
|
||||||
cleanup obligation as an unrelated side effect.
|
failure leaves publish succeeded and the remote committed snapshot current.
|
||||||
|
Retry with a normal full run: valid upstream work, including publish and notify,
|
||||||
|
is skipped while the failed cleanup stage runs again. `run-stage cleanup` is
|
||||||
|
also available when the manifest has valid publish evidence. A range ending at
|
||||||
|
`notify` or earlier and the standalone `publish` command never delete local
|
||||||
|
state.
|
||||||
|
|
||||||
|
Older local manifests that contain the retired cleanup-obligation field are
|
||||||
|
intentionally incompatible with strict manifest decoding. If such an
|
||||||
|
obligation is still outstanding, complete it with the pre-upgrade binary.
|
||||||
|
Otherwise verify that the remote publication is committed, move aside or
|
||||||
|
regenerate the incompatible local manifest (or restore the verified remote
|
||||||
|
snapshot), and then use the cleanup stage. Manual JSON editing is not the
|
||||||
|
normal migration path.
|
||||||
|
|
||||||
## Operational Caveats
|
## Operational Caveats
|
||||||
|
|
||||||
@@ -571,4 +757,5 @@ cleanup obligation as an unrelated side effect.
|
|||||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||||
- Restore requires configured object storage and committed remote current state.
|
- Restore requires configured object storage and committed remote current state.
|
||||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
|
||||||
|
resolution and object-store initialization.
|
||||||
|
|||||||
@@ -116,8 +116,11 @@ adapter and publish metadata.
|
|||||||
|
|
||||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||||
inspected and validated, but file presence alone does not replace recorded
|
inspected and validated, but file presence alone does not replace recorded
|
||||||
stage state. Invocation-scoped run records provide an audit of one execution;
|
stage state. Each invocation retains an in-memory command summary and a
|
||||||
they do not replace the session manifest as progress authority.
|
run-local workspace identity, but neither is a durable progress ledger. The
|
||||||
|
physical run directory is created on demand only when an executing stage
|
||||||
|
requires it; a fully reused invocation or a cleanup-only invocation does not
|
||||||
|
leave an empty run directory.
|
||||||
|
|
||||||
Restore treats committed remote current state as its authority. It must plan
|
Restore treats committed remote current state as its authority. It must plan
|
||||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||||
@@ -143,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
|
|||||||
- YAML decoding rejects unknown fields.
|
- YAML decoding rejects unknown fields.
|
||||||
- Defaults are centralized and testable.
|
- Defaults are centralized and testable.
|
||||||
- Empty configured values do not silently replace meaningful defaults.
|
- Empty configured values do not silently replace meaningful defaults.
|
||||||
- Validation rejects invalid composition before stage execution where
|
- Validation rejects invalid configuration before stage execution where
|
||||||
practical.
|
practical.
|
||||||
- Root-owned imports and one selected profile resolve deterministically through
|
- Root-owned imports and one selected profile resolve deterministically through
|
||||||
the configuration owner; commands do not implement their own merge rules.
|
the configuration owner; commands do not implement their own merge rules.
|
||||||
@@ -185,8 +188,8 @@ availability.
|
|||||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||||
relative destinations must not expand the scope of an operation. Cleanup is
|
relative destinations must not expand the scope of an operation. Cleanup is
|
||||||
permitted only through explicit operator action or configured post-publish
|
permitted only through explicit operator action or the configured final cleanup
|
||||||
gates, and it must preserve durable cache unless cache removal is explicitly
|
stage, and it must preserve durable cache unless cache removal is explicitly
|
||||||
requested.
|
requested.
|
||||||
|
|
||||||
Physical layout, retention, and operational lifecycle belong in
|
Physical layout, retention, and operational lifecycle belong in
|
||||||
@@ -202,16 +205,26 @@ the immutable commit manifest, and finally the current commit pointer.
|
|||||||
`current/commit-pointer.json` is the sole mutable selector and must be written
|
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||||
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||||
must not be presented as current remote state. Publish locks remain authoritative
|
must not be presented as current remote state. Publish locks remain authoritative
|
||||||
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
|
and are not bypassed by a forced run. Each mutable remote lock is an independent
|
||||||
generation preconditions and are revalidated immediately before pointer
|
per-source object; lock mutations use provider-enforced generation preconditions.
|
||||||
selection; loss of that check leaves the prior committed snapshot current.
|
Publish revalidates every initially unlocked participating source after writing
|
||||||
|
the immutable commit and before pointer selection. A lock observed then leaves
|
||||||
|
the prior committed snapshot current. A lock created after that final check may
|
||||||
|
race the pointer write; the commit protocol does not claim a stronger distributed
|
||||||
|
transaction boundary.
|
||||||
|
|
||||||
Automatic local cleanup is permitted only after a successful publish commit,
|
Automatic local cleanup is permitted only after a successful publish commit,
|
||||||
only when explicitly configured, and only through the path-safety guardrails.
|
only when explicitly configured, and only through the path-safety guardrails.
|
||||||
It is a durable local obligation bound to that committed run and its exact
|
It is the final ordinary pipeline stage after `notify`. The stage derives its
|
||||||
targets, not an inferred side effect of the current stage list. A cleanup
|
exact run-scoped targets from validated publish metadata and the configured
|
||||||
failure makes the invocation incomplete while leaving the committed remote
|
campaign/session identity; it never infers a target from directory contents or
|
||||||
snapshot authoritative; later invocations resume the recorded obligation.
|
rereads remote storage. A cleanup failure makes the invocation incomplete while
|
||||||
|
leaving publish succeeded and the committed remote snapshot authoritative. A
|
||||||
|
normal later full run reuses valid upstream stages and retries cleanup.
|
||||||
|
|
||||||
|
The standalone `publish` command selects only publish and therefore never
|
||||||
|
invokes cleanup. Operators may select cleanup directly only when the manifest
|
||||||
|
contains valid successful publication evidence.
|
||||||
|
|
||||||
## Security, Privacy, And Diagnostics
|
## Security, Privacy, And Diagnostics
|
||||||
|
|
||||||
@@ -221,20 +234,26 @@ configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
|||||||
intentionally shareable with the configured workspace group. API-key material
|
intentionally shareable with the configured workspace group. API-key material
|
||||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||||
|
|
||||||
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
|
||||||
|
a cross-build gate and release asset, but no native runtime release gate.
|
||||||
|
Windows and macOS amd64 are unsupported and have no compatibility promise or
|
||||||
|
release asset.
|
||||||
|
|
||||||
|
On supported systems, Narratio-created ordinary workspace directories converge on
|
||||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||||
umask is restrictive. This preserves the existing workspace group for nested
|
umask is restrictive. This preserves the existing workspace group for nested
|
||||||
creation and atomic replacements without changing ownership. API-key storage
|
creation and atomic replacements without changing ownership. API-key storage
|
||||||
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
uses a separate restrictive contract.
|
||||||
are not authoritative; operators must provide the equivalent shared-group and
|
|
||||||
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
|
||||||
|
|
||||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||||
to manifests, logs, generated configuration, reports, publish metadata,
|
to manifests, logs, generated configuration, reports, publish metadata,
|
||||||
documentation, or examples. Secrets enter through configured environment
|
documentation, or examples. Pipeline configuration names environment variables;
|
||||||
variable names or secret-file references. Diagnostics should avoid transcript
|
command-owned startup may populate previously unset variables from ConfigKit's
|
||||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
explicit or conventional secrets directory. This bootstrap occurs once per
|
||||||
it.
|
invocation before any typed configuration load or adapter construction and is
|
||||||
|
not hidden inside library loaders. Safe bootstrap diagnostics may identify
|
||||||
|
files and statuses but never values. Diagnostics should avoid transcript and
|
||||||
|
prompt content unless a deliberate, bounded inspection mechanism requires it.
|
||||||
|
|
||||||
Logs, reports, generated invocation files, generated configuration, and render
|
Logs, reports, generated invocation files, generated configuration, and render
|
||||||
debug files are diagnostics, not canonical pipeline products. They should be
|
debug files are diagnostics, not canonical pipeline products. They should be
|
||||||
|
|||||||
@@ -5,6 +5,13 @@ binary release. The synchronous release boundary is a successful push of one
|
|||||||
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
|
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
|
||||||
change that result.
|
change that result.
|
||||||
|
|
||||||
|
## Go prerequisite
|
||||||
|
|
||||||
|
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
|
||||||
|
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
|
||||||
|
asset construction; that asynchronous work does not change the tag-push
|
||||||
|
completion boundary.
|
||||||
|
|
||||||
## Choose a version and write its note
|
## Choose a version and write its note
|
||||||
|
|
||||||
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
|
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
|
||||||
@@ -51,14 +58,14 @@ scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
|||||||
|
|
||||||
It validates the version and matching note, module hygiene, formatting,
|
It validates the version and matching note, module hygiene, formatting,
|
||||||
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
||||||
and six official cross-build assets. It uses `GOWORK=off`, does not contact
|
and three official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||||
application services, CI, or Gitea, and does not create tags or modify tracked
|
application services, CI, or Gitea, and does not create tags or modify tracked
|
||||||
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
||||||
checker.
|
checker.
|
||||||
|
|
||||||
The checker builds Linux, macOS, and Windows assets for `amd64` and `arm64`.
|
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
|
||||||
Cross-builds prove compilation; they are not native macOS or Windows runtime
|
`arm64`. The macOS cross-build proves compilation; it is not native macOS
|
||||||
evidence.
|
runtime evidence.
|
||||||
|
|
||||||
## Publish the tag
|
## Publish the tag
|
||||||
|
|
||||||
|
|||||||
@@ -22,5 +22,7 @@ See the [release procedure](../release.md) for creating a candidate and tag.
|
|||||||
When asynchronous publication succeeds, the corresponding Gitea release is the
|
When asynchronous publication succeeds, the corresponding Gitea release is the
|
||||||
canonical source for downloadable binaries and checksums.
|
canonical source for downloadable binaries and checksums.
|
||||||
|
|
||||||
|
- [v2.0.0](v2.0.0.md)
|
||||||
|
- [v1.7.0](v1.7.0.md)
|
||||||
- [v1.6.0](v1.6.0.md)
|
- [v1.6.0](v1.6.0.md)
|
||||||
- [v1.5.0](v1.5.0.md)
|
- [v1.5.0](v1.5.0.md)
|
||||||
|
|||||||
100
docs/releases/v1.7.0.md
Normal file
100
docs/releases/v1.7.0.md
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# Narratio v1.7.0
|
||||||
|
|
||||||
|
Narratio v1.7.0 completes the maintainability refactor begun after v1.6.0. It
|
||||||
|
simplifies filesystem and process handling, makes resumable work more precise,
|
||||||
|
replaces legacy local and remote state with explicit authorities, and turns
|
||||||
|
cleanup into the final ordinary pipeline stage.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
The session manifest is now the sole durable local progress ledger. Analyze
|
||||||
|
checkpoints successful artifacts individually, allowing an interrupted or
|
||||||
|
partially failed analysis to resume at the first incomplete artifact without
|
||||||
|
rerunning successful siblings unless forced.
|
||||||
|
|
||||||
|
Remote publication now commits immutable run state before advancing one
|
||||||
|
verified current pointer. Publish locks are independent per-source objects;
|
||||||
|
locked outputs retain their previously committed declarations rather than
|
||||||
|
being overwritten or silently omitted. Local cleanup runs after notification
|
||||||
|
as a separately recorded, safely retryable stage.
|
||||||
|
|
||||||
|
The release also adopts Go 1.26, simplifies ordinary file operations around
|
||||||
|
Narratio's trusted group-owned workspace model, improves subprocess lifecycle
|
||||||
|
handling, decomposes several large orchestration and validation components,
|
||||||
|
and removes mutable global test seams.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a minor release with deliberate removal of compatibility paths that
|
||||||
|
were deprecated and documented as subject to removal:
|
||||||
|
|
||||||
|
- Campaigns must use one versioned `narratio.party.v1` `party.yml`. Separate
|
||||||
|
players files, session-level party overrides, and unversioned party documents
|
||||||
|
are rejected.
|
||||||
|
- Session manifests must declare `schema_version:
|
||||||
|
narratio.session-manifest.v1` and satisfy the current strict schema. Older
|
||||||
|
unversioned state and manifests containing the retired
|
||||||
|
`post_publish_cleanup` obligation are rejected rather than migrated.
|
||||||
|
- Remote current state must be selected by
|
||||||
|
`current/commit-pointer.json` and its referenced immutable run commit. Legacy
|
||||||
|
current-state layouts are no longer inferred.
|
||||||
|
- The retired aggregate `locks.yml` object is ignored. Publish locks now live
|
||||||
|
as independent strict objects under the session lock prefix.
|
||||||
|
- Run manifests are no longer produced or consumed. The session manifest owns
|
||||||
|
durable stage progress; invocation summaries and run workspaces remain
|
||||||
|
operational data rather than a second progress authority.
|
||||||
|
|
||||||
|
The CLI and current pipeline/configuration formats otherwise remain compatible.
|
||||||
|
Building and maintaining Narratio now requires Go 1.26. Cross-built Linux,
|
||||||
|
macOS, and Windows assets remain available for `amd64` and `arm64`.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Before upgrading a campaign that still uses compatibility data:
|
||||||
|
|
||||||
|
1. Replace legacy party/player inputs with the documented
|
||||||
|
[`narratio.party.v1`](../integrations/party.md) campaign roster.
|
||||||
|
2. If a local manifest contains an outstanding `post_publish_cleanup`
|
||||||
|
obligation, complete it with the pre-upgrade binary. Otherwise verify the
|
||||||
|
remote commit, move aside or regenerate the incompatible manifest, or
|
||||||
|
restore a verified committed snapshot. Do not migrate state by manually
|
||||||
|
editing JSON.
|
||||||
|
3. Republish sessions that do not yet have a valid immutable run commit and
|
||||||
|
`current/commit-pointer.json`; legacy remote layouts are not current state in
|
||||||
|
v1.7.0.
|
||||||
|
4. Recreate any required publication locks with `narratio session locks add`.
|
||||||
|
After verifying the new locks and publication behavior, the ignored legacy
|
||||||
|
`locks.yml` object may be removed manually.
|
||||||
|
|
||||||
|
Existing compatible session manifests and already committed remote snapshots
|
||||||
|
need no special action. Consult [Operations](../operations.md) and
|
||||||
|
[Troubleshooting](../troubleshooting.md) for recovery procedures.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Upgraded the maintained development and release baseline to Go 1.26.
|
||||||
|
- Reworked subprocess execution and Windows process-tree supervision while
|
||||||
|
preserving contextual wait and cleanup failures.
|
||||||
|
- Replaced custom confined ordinary I/O and directory-promotion machinery with
|
||||||
|
simpler standard-library operations while retaining bounded reads, atomic
|
||||||
|
single-file visibility, workspace modes, validated downloads, and
|
||||||
|
root-confined destructive operations.
|
||||||
|
- Removed legacy party, manifest, artifact-discovery, remote-current, and lock
|
||||||
|
compatibility paths.
|
||||||
|
- Consolidated durable progress in the session manifest and removed the run
|
||||||
|
manifest domain model.
|
||||||
|
- Unified runner planning and execution decisions, separated invocation
|
||||||
|
lifecycle responsibilities, and avoided creating empty run workspaces for
|
||||||
|
fully reused or cleanup-only invocations.
|
||||||
|
- Added per-artifact analyze execution, checkpointing, reconciliation, and
|
||||||
|
recovery.
|
||||||
|
- Decomposed configuration validation by domain and made validation diagnostics
|
||||||
|
deterministic without treating incidental ordering or wording as a contract.
|
||||||
|
- Introduced immutable remote publish commits, one final current pointer,
|
||||||
|
explicit output planning, safe cross-run carried outputs, and independent
|
||||||
|
per-artifact remote locks.
|
||||||
|
- Added `cleanup` as the final canonical stage after `notify`, with validated
|
||||||
|
publish evidence, safe retry, explicit selection, and no separate cleanup
|
||||||
|
obligation.
|
||||||
|
- Replaced mutable command test hooks with invocation-scoped dependencies and
|
||||||
|
improved structural test diagnostics.
|
||||||
94
docs/releases/v2.0.0.md
Normal file
94
docs/releases/v2.0.0.md
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
# Narratio v2.0.0
|
||||||
|
|
||||||
|
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
|
||||||
|
simplifies the supported release platforms, and makes artifact definitions and
|
||||||
|
their effective dependency graph directly inspectable.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
|
||||||
|
imports, fixed-path named profiles, provenance, defaults, and typed credential
|
||||||
|
references. The recommended production layout keeps production values in the
|
||||||
|
base, defines each ordinary artifact and artifact family in its own imported
|
||||||
|
file, and keeps testing changes in `profiles/testing.yml`.
|
||||||
|
|
||||||
|
Configured-artifact inputs now create their dependency edges automatically.
|
||||||
|
`narratio config artifacts` reports the finalized concrete catalog, including
|
||||||
|
generated party-family members, without loading a session or initializing
|
||||||
|
runtime adapters.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a breaking configuration and platform release. There is no runtime
|
||||||
|
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
|
||||||
|
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
|
||||||
|
presence/default behavior. Defaults apply only to absent values; explicit empty,
|
||||||
|
zero, false, and null values retain their typed meaning and may fail Narratio
|
||||||
|
domain validation.
|
||||||
|
|
||||||
|
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
|
||||||
|
and macOS amd64 builds are no longer produced or supported. Existing v1 release
|
||||||
|
notes remain historical descriptions of their releases.
|
||||||
|
|
||||||
|
Inspection commands still avoid session, manifest, workspace, adapter, and
|
||||||
|
remote-service activity, but they are not credential-free loaders: every
|
||||||
|
explicit typed credential reference in the selected effective configuration must
|
||||||
|
resolve; references replaced or removed by that profile need not resolve.
|
||||||
|
Pipelines containing artifact families also require a campaign so the canonical
|
||||||
|
party can be expanded.
|
||||||
|
`config show` is a secret-safe review projection, not a promised reloadable
|
||||||
|
authoring file.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Migrate each operational bundle before running v2:
|
||||||
|
|
||||||
|
1. Replace old composition syntax with root-owned
|
||||||
|
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
|
||||||
|
additive base sources; they do not override one another.
|
||||||
|
2. Fold the old production overlay into the base pipeline. Move testing changes
|
||||||
|
to `profiles/testing.yml`; compare the two independent resolutions with
|
||||||
|
`narratio config diff default testing`.
|
||||||
|
3. Follow the maintained
|
||||||
|
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
|
||||||
|
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
|
||||||
|
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
|
||||||
|
stable artifact identity.
|
||||||
|
4. Move credential files to an untracked `secrets/` directory beside
|
||||||
|
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
|
||||||
|
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
|
||||||
|
5. For static S3 credentials, configure both S3 references (commonly
|
||||||
|
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
|
||||||
|
To use instance roles, workload identity, shared SDK files, or another AWS SDK
|
||||||
|
discovery provider, omit both references. A partial pair is invalid.
|
||||||
|
6. Remove a `depends_on` entry when it only duplicates a
|
||||||
|
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
|
||||||
|
without supplying the upstream output as an input.
|
||||||
|
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
|
||||||
|
that previously acquired defaults after being explicitly empty. Do not use
|
||||||
|
`config show` output as a source migration mechanism.
|
||||||
|
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
|
||||||
|
with the same profile, campaign, and secrets selection intended for runtime.
|
||||||
|
Resolve the campaign-owned party whenever families are present.
|
||||||
|
9. Update deployment and download automation to one of the three maintained
|
||||||
|
platform assets before removing the v1 binary.
|
||||||
|
|
||||||
|
The [configuration reference](../config.md), [CLI reference](../cli.md), and
|
||||||
|
[maintained examples](../../examples/README.md) are the canonical detailed
|
||||||
|
guides.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
|
||||||
|
while keeping Narratio domain validation and post-campaign expansion explicit.
|
||||||
|
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
|
||||||
|
unified safe effective show/diff/digest projection, and source provenance.
|
||||||
|
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
|
||||||
|
Audita credential references without exposing resolved values.
|
||||||
|
- Inferred, normalized, and validated configured-artifact input dependencies as
|
||||||
|
part of the one concrete graph consumed by analyze planning and resume logic.
|
||||||
|
- Added `config artifacts` with stable concrete identities, definition sources,
|
||||||
|
effective dependencies, and generated family/party provenance.
|
||||||
|
- Added the maintained production-base/testing-overlay artifact authoring layout.
|
||||||
|
- Reduced release assets and active platform support to Linux amd64/arm64 and
|
||||||
|
macOS arm64.
|
||||||
70
docs/releases/v2.1.0.md
Normal file
70
docs/releases/v2.1.0.md
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
# Narratio v2.1.0
|
||||||
|
|
||||||
|
This release adds explicit Notarius credential forwarding and cache reuse,
|
||||||
|
with predictable forced regeneration and clearer artifact-development guidance.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
|
||||||
|
its accepted execution checkpoints. Stable, verified reference snapshots allow
|
||||||
|
unchanged references to retain their cache identity across Narratio invocations.
|
||||||
|
Configuration loading now uses ConfigKit v0.3.0.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
|
||||||
|
fields are optional. Existing v2 configuration needs no schema migration.
|
||||||
|
Configured credential references must resolve during configuration loading,
|
||||||
|
even when extraction will not execute; omitted references add no requirement.
|
||||||
|
|
||||||
|
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
|
||||||
|
refresh. This includes `regenerate-artifacts`, which remains an alias for a
|
||||||
|
forced run from extract through analyze. `analyze` continues to force analysis
|
||||||
|
only, preserving extraction results and transcripts.
|
||||||
|
|
||||||
|
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
|
||||||
|
minimum-version gate. The executable must support the flags selected for the
|
||||||
|
invocation, including `--chunk_cache refresh` for forced extraction. Reference
|
||||||
|
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
|
||||||
|
schemas are unchanged; reused producers may report zero production attempts.
|
||||||
|
|
||||||
|
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
|
||||||
|
arm64. macOS validation is cross-compilation, not native runtime testing.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special migration is required for existing v2 configuration. To use the new
|
||||||
|
features:
|
||||||
|
|
||||||
|
- Declare `notarius.llm_api_key` with the environment-variable name expected by
|
||||||
|
the deployed Notarius profile, supplying its value through the existing
|
||||||
|
environment or secrets-directory mechanism.
|
||||||
|
- Enable checkpoint recording in Notarius configuration before setting
|
||||||
|
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
|
||||||
|
roots for service deployments. An optional `notarius.chunk_cache` overrides
|
||||||
|
its ordinary chunk-plan policy.
|
||||||
|
- Expect an initial checkpoint miss when moving from old run-local reference
|
||||||
|
paths to stable session snapshots. No migration or deletion of old caches is
|
||||||
|
required. Force refreshes the current input's plan, not the whole cache.
|
||||||
|
- Use `narratio analyze SESSION` for analysis-only iteration, and
|
||||||
|
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
|
||||||
|
|
||||||
|
See [Configuration](../config.md), the
|
||||||
|
[Notarius integration contract](../integrations/notarius.md), and
|
||||||
|
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
|
||||||
|
for current details.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
|
||||||
|
from machine configuration.
|
||||||
|
- Added optional typed Notarius credentials, exact-name environment delivery,
|
||||||
|
safe configuration projections, and captured-stream redaction for custom names.
|
||||||
|
- Added Notarius resume and chunk-cache controls with force taking precedence.
|
||||||
|
- Added verified, content-addressed session reference snapshots that survive
|
||||||
|
ordinary run cleanup without becoming published artifacts.
|
||||||
|
- Preserved relative-workspace support for snapshot delivery and confinement.
|
||||||
|
- Kept credentials and cache-policy controls outside extract's semantic resume
|
||||||
|
identity, so operational changes alone do not invalidate completed results.
|
||||||
|
- Added an opt-in real Notarius integration test using a local provider to
|
||||||
|
verify reuse, partial-failure recovery, and forced refresh without paid calls.
|
||||||
33
docs/releases/v2.1.1.md
Normal file
33
docs/releases/v2.1.1.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Narratio v2.1.1
|
||||||
|
|
||||||
|
This release fixes configuration loading for publish rules that explicitly set
|
||||||
|
requiredness while relying on a derived destination.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Publish rules with a derivable source and omitted `dest` now load correctly
|
||||||
|
whether `required` is omitted, explicitly true, or explicitly false. Previously,
|
||||||
|
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
|
||||||
|
diagnostic instead of accepting valid configuration.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a backward-compatible bookkeeping correction. Publish schemas,
|
||||||
|
destination derivation, requiredness defaults, and YAML syntax support are
|
||||||
|
unchanged. Explicit destinations and explicit false values remain intact.
|
||||||
|
No ConfigKit upgrade or publication/storage behavior change is involved.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special upgrade action or configuration migration is required. Operators
|
||||||
|
need not add a derivable `dest`, remove explicit `required` values, or change
|
||||||
|
flow-style YAML to block-style YAML. See the
|
||||||
|
[configuration reference](../config.md#publish-configuration-summary).
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
|
||||||
|
while retaining field-level resolution records.
|
||||||
|
- Added real load/validation and safe effective-configuration regression coverage
|
||||||
|
for built-in and configured-artifact sources, explicit requiredness, and an
|
||||||
|
explicit destination that must remain unchanged.
|
||||||
26
docs/releases/v2.1.2.md
Normal file
26
docs/releases/v2.1.2.md
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# Narratio v2.1.2
|
||||||
|
|
||||||
|
This release allows secrets directories to contain hidden metadata without
|
||||||
|
preventing credential loading.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
|
||||||
|
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
|
||||||
|
valid credentials from loading and are omitted from bootstrap reports.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a backward-compatible dependency bug fix. Configuration schemas,
|
||||||
|
existing-environment precedence, and validation of non-hidden entries are
|
||||||
|
unchanged. Hidden entries still count toward the directory entry limit.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special upgrade action or configuration migration is required. See the
|
||||||
|
[credential configuration reference](../config.md) for secrets-directory usage.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Updated ConfigKit from v0.3.0 to v0.3.1.
|
||||||
|
- Clarified hidden-entry handling in the secrets-directory documentation.
|
||||||
53
docs/releases/v2.2.0.md
Normal file
53
docs/releases/v2.2.0.md
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
# Narratio v2.2.0
|
||||||
|
|
||||||
|
This release adds resumable per-speaker transcription, replayable WhisperX
|
||||||
|
uploads, and explicit Notarius configuration profile selection.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Successfully checkpointed speaker transcripts survive later transcription
|
||||||
|
failures. Ordinary retry reuses valid completed speakers and processes only the
|
||||||
|
remaining work. WhisperX uploads can now be replayed by the HTTP transport
|
||||||
|
without buffering whole audio files.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing CLI commands, speaker identities, transcript formats, and pipeline
|
||||||
|
ordering are unchanged. Session manifests gain optional per-speaker completion
|
||||||
|
evidence. Older manifests remain readable, but older Narratio binaries may
|
||||||
|
reject manifests containing the new fields; downgrading after transcription
|
||||||
|
is not transparent.
|
||||||
|
|
||||||
|
The optional `pipeline.notarius.config_profile` field selects a downstream
|
||||||
|
Notarius configuration profile independently of Narratio's own profile. Using
|
||||||
|
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
|
||||||
|
supporting version. Omitting the field retains downstream default selection.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No configuration migration is required. When transcribe is next selected for
|
||||||
|
an older aggregate-only manifest, all current speakers are transcribed once
|
||||||
|
to establish reusable evidence. Previously unrecorded results from failed runs
|
||||||
|
are not automatically imported.
|
||||||
|
|
||||||
|
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
|
||||||
|
reuse of prior results; an ordinary retry after a failed forced invocation can
|
||||||
|
retain replacements already checkpointed by that attempt. See the
|
||||||
|
[operations guide](../operations.md) and
|
||||||
|
[Notarius profile configuration](../config.md#notarius-configuration-profile).
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Persist validated speaker completions independently of aggregate stage status,
|
||||||
|
with serialized saves for concurrent work and input/output integrity checks.
|
||||||
|
- Rebuild aggregate transcript references when speakers are removed or those
|
||||||
|
references need repair, without retranscribing otherwise-current speakers.
|
||||||
|
- Keep removed and incomplete speaker results out of versioned merge inputs.
|
||||||
|
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
|
||||||
|
service rejection still fails normally.
|
||||||
|
- Provide fresh streaming multipart bodies for HTTP transport replay while
|
||||||
|
preserving application retries, cancellation, and resource cleanup. This
|
||||||
|
addresses the missing-body-replay limitation, not persistent upstream
|
||||||
|
protocol failures or exactly-once service execution.
|
||||||
|
- Pass explicit Notarius configuration profiles through the CLI and include
|
||||||
|
their selection in extraction reuse decisions.
|
||||||
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
---
|
||||||
|
title: Select a Notarius configuration profile through Narratio
|
||||||
|
status: proposed
|
||||||
|
consumer: rakestraw.dnd Ansible collection and its pipeline role
|
||||||
|
|
||||||
|
use_case: >-
|
||||||
|
The Ansible pipeline role deploys Narratio and Notarius as independently
|
||||||
|
configured applications. Operators select Narratio's testing profile to
|
||||||
|
develop prompts and other pipeline assets with inexpensive models while
|
||||||
|
retaining production settings in the base configuration. That selection
|
||||||
|
currently cannot select a Notarius configuration profile, leaving extraction
|
||||||
|
on its production settings.
|
||||||
|
|
||||||
|
desired_outcome: >-
|
||||||
|
Narratio's testing overlay can explicitly select Notarius's testing overlay.
|
||||||
|
Notarius owns the resulting model choices and any other testing-specific
|
||||||
|
settings. The operator selects testing once when invoking Narratio, without
|
||||||
|
editing production files or separately invoking Notarius.
|
||||||
|
|
||||||
|
rationale: >-
|
||||||
|
Both applications already use ConfigKit configuration overlays. Selecting
|
||||||
|
the downstream overlay preserves each application's configuration ownership
|
||||||
|
and supports future Notarius testing adjustments without adding a Narratio
|
||||||
|
configuration field for every downstream setting. This is explicit profile
|
||||||
|
composition, not a requirement for arbitrary recursive orchestration.
|
||||||
|
|
||||||
|
requested_interface:
|
||||||
|
field: pipeline.notarius.config_profile
|
||||||
|
value: Optional Notarius ConfigKit configuration profile name
|
||||||
|
supplied: >-
|
||||||
|
Invoke Notarius with --config-profile and the selected name when extraction
|
||||||
|
executes. The setting is available in Narratio's base configuration and
|
||||||
|
its ordinary configuration overlays.
|
||||||
|
omitted_or_null: >-
|
||||||
|
Do not supply a profile-selection flag. Preserve existing Notarius
|
||||||
|
configuration selection behavior.
|
||||||
|
downstream_semantics: >-
|
||||||
|
Retain Notarius's existing selector semantics, including the default
|
||||||
|
selector for its base configuration. Notarius resolves profiles relative
|
||||||
|
to its own configuration root, not Narratio's root.
|
||||||
|
explicit_selection: >-
|
||||||
|
Do not automatically forward Narratio's profile name. The selected
|
||||||
|
Notarius profile may have the same name or a different name.
|
||||||
|
distinction: >-
|
||||||
|
This selects a ConfigKit configuration overlay through --config-profile,
|
||||||
|
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
|
||||||
|
can itself select the desired LLM profiles using native Notarius settings.
|
||||||
|
|
||||||
|
illustrative_consumer_configuration:
|
||||||
|
note: >-
|
||||||
|
These are separate files managed by the Ansible suite, not a proposed
|
||||||
|
combined configuration document. The Notarius pipeline and LLM profile
|
||||||
|
names are illustrative; the referenced LLM profile must be available to
|
||||||
|
Notarius.
|
||||||
|
narratio_profiles_testing_yml: |
|
||||||
|
notarius:
|
||||||
|
config_profile: testing
|
||||||
|
notarius_profiles_testing_yml: |
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
llm_profile: gemini-flash-latest
|
||||||
|
operator_workflow: >-
|
||||||
|
Select --profile testing on the ordinary Narratio invocation. Narratio
|
||||||
|
selects Notarius's testing configuration overlay when it runs extraction;
|
||||||
|
other Narratio testing overrides continue to apply normally.
|
||||||
|
|
||||||
|
observable_behavior:
|
||||||
|
- >-
|
||||||
|
Notarius owns overlay discovery, composition, validation, and the effective
|
||||||
|
downstream settings. A missing or invalid selected overlay must not silently
|
||||||
|
fall back to production or be reported as successful extraction.
|
||||||
|
- >-
|
||||||
|
Changing the selected Notarius configuration profile must not silently
|
||||||
|
reuse Narratio's completed extraction from the previous selection.
|
||||||
|
Existing downstream invalidation behavior applies when extraction changes.
|
||||||
|
- >-
|
||||||
|
Editing Notarius overlay contents behind an unchanged selector retains
|
||||||
|
the existing operator responsibility to force extraction when needed.
|
||||||
|
This request does not require inspecting or tracking private Notarius files.
|
||||||
|
- >-
|
||||||
|
Existing credential forwarding remains independently configurable through
|
||||||
|
notarius.llm_api_key, including in the Narratio testing overlay when the
|
||||||
|
selected Notarius settings require a different credential reference.
|
||||||
|
|
||||||
|
consumer_follow_up: >-
|
||||||
|
After Narratio releases this interface, the standalone Narratio Ansible role
|
||||||
|
can expose an optional variable and omit the field when unset. The suite role
|
||||||
|
can deploy Notarius's testing overlay and select it from Narratio's testing
|
||||||
|
overlay, then distribute the changes through the collection. These consumer
|
||||||
|
updates are separate from this upstream feature request.
|
||||||
|
|
||||||
|
scope_boundaries:
|
||||||
|
- No Notarius change is currently required; v0.7.0 supports --config-profile.
|
||||||
|
- No separate Narratio-side Notarius LLM-profile selector is requested.
|
||||||
|
- No automatic profile-name propagation or downstream configuration rewriting.
|
||||||
|
- No new credential discovery, copying, or provider-name remapping.
|
||||||
|
- This request specifies consumer behavior, not internal Narratio code changes.
|
||||||
|
|
||||||
|
references:
|
||||||
|
narratio_configuration: ../config.md
|
||||||
|
narratio_notarius_integration: ../integrations/notarius.md
|
||||||
|
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md
|
||||||
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
|
||||||
|
|
||||||
|
Status: Implemented; release target v2.1.1
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Narratio should successfully derive a publish destination when a valid
|
||||||
|
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
|
||||||
|
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
|
||||||
|
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
|
||||||
|
|
||||||
|
This is a configuration-resolution bookkeeping defect, not a YAML formatting
|
||||||
|
restriction. The fix should preserve the existing public configuration
|
||||||
|
contract: operators may omit a derivable destination and may independently set
|
||||||
|
`required` to either `true` or `false`.
|
||||||
|
|
||||||
|
## Observed Behavior
|
||||||
|
|
||||||
|
A pipeline containing a rule equivalent to the following fails during native
|
||||||
|
configuration loading:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
outputs: [{"source": "narratio.transcript.final", "required": true}]
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio reports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
|
||||||
|
```
|
||||||
|
|
||||||
|
The same failure is possible with ordinary block-style YAML when `required` is
|
||||||
|
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
|
||||||
|
cause of the defect.
|
||||||
|
|
||||||
|
The existing destination-derivation tests do not expose this case because
|
||||||
|
their rules also omit `required`. Narratio then supplies the default value for
|
||||||
|
`required` and records a change covering the complete `publish.outputs`
|
||||||
|
sequence. That broader record happens to cover the destination mutation as
|
||||||
|
well, masking the missing resolution record that occurs when `required` is
|
||||||
|
already explicit.
|
||||||
|
|
||||||
|
## Expected Contract
|
||||||
|
|
||||||
|
The current configuration contract remains unchanged:
|
||||||
|
|
||||||
|
- `publish.outputs[].source` is required;
|
||||||
|
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
|
||||||
|
selected built-in or configured artifact source;
|
||||||
|
- `publish.outputs[].required` defaults to `true`, but an operator may set it
|
||||||
|
explicitly to either `true` or `false`; and
|
||||||
|
- equivalent valid YAML representations have equivalent configuration
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
For example, both rules below should load and validate, retain their explicit
|
||||||
|
requiredness, and receive the canonical derived destination:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
required: false
|
||||||
|
```
|
||||||
|
|
||||||
|
## Root Cause
|
||||||
|
|
||||||
|
ConfigKit intentionally treats sequences as atomic when comparing values
|
||||||
|
before and after application resolution. Changing a field within one
|
||||||
|
`publish.outputs` entry is therefore observed as a change to the
|
||||||
|
`/publish/outputs` sequence.
|
||||||
|
|
||||||
|
Narratio currently derives each missing destination by recording a
|
||||||
|
normalization at the individual `publish.outputs[index].dest` path. That child
|
||||||
|
record does not cover the atomic parent-sequence change. A separate
|
||||||
|
container-level record is currently created only when Narratio also defaults
|
||||||
|
an omitted `required` value, so configurations with explicit `required` values
|
||||||
|
lose that incidental coverage.
|
||||||
|
|
||||||
|
## Recommended Fix
|
||||||
|
|
||||||
|
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
|
||||||
|
atomic-sequence contract. Whenever resolution derives one or more destinations
|
||||||
|
inside `publish.outputs`, the recorded resolution changes must cover the
|
||||||
|
`/publish/outputs` container. Destination derivation must not depend on an
|
||||||
|
unrelated `required` default being applied in the same list.
|
||||||
|
|
||||||
|
The implementation may retain more specific provenance where useful, but it
|
||||||
|
must provide valid container-level coverage for the sequence mutation. Do not
|
||||||
|
change ConfigKit's sequence comparison semantics merely to accommodate this
|
||||||
|
call site.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- Correct destination derivation for publish rules that explicitly set
|
||||||
|
`required` and omit a derivable `dest`.
|
||||||
|
- Preserve explicit `required: true` and `required: false` values.
|
||||||
|
- Preserve canonical destination derivation for built-in transcript sources
|
||||||
|
and configured artifact sources.
|
||||||
|
- Add focused regression coverage through Narratio's real configuration load
|
||||||
|
and validation boundary.
|
||||||
|
- Prepare a backward-compatible patch release after the fix passes the normal
|
||||||
|
repository validation.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
|
||||||
|
destination derivation rules.
|
||||||
|
- Requiring operators or configuration generators to supply `dest` when it is
|
||||||
|
derivable.
|
||||||
|
- Requiring operators to omit explicit `required: true` values.
|
||||||
|
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
|
||||||
|
- Changing ConfigKit's documented atomic treatment of sequences.
|
||||||
|
- Adding storage, publication, or network integration behavior.
|
||||||
|
|
||||||
|
## Regression Coverage
|
||||||
|
|
||||||
|
Add one focused package-level behavioral test that loads and validates a
|
||||||
|
pipeline with omitted destinations and explicit requiredness. A representative
|
||||||
|
list should include both `required: true` and `required: false`, then assert
|
||||||
|
that:
|
||||||
|
|
||||||
|
- loading and validation succeed;
|
||||||
|
- each destination is derived from its source; and
|
||||||
|
- each explicit `required` value is preserved.
|
||||||
|
|
||||||
|
Using the compact flow-style representation that exposed the integration
|
||||||
|
failure is reasonable, while the existing block-style cases continue to cover
|
||||||
|
omitted `required`. The test should exercise the public configuration behavior,
|
||||||
|
not ConfigKit call choreography or Narratio's private recording mechanism.
|
||||||
|
|
||||||
|
No network service, object store, credential, or test double is required.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
All valid publish rules with derivable sources behave consistently across
|
||||||
|
these cases:
|
||||||
|
|
||||||
|
| `dest` | `required` | Result |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| omitted | omitted | destination derived; required defaults to `true` |
|
||||||
|
| omitted | `true` | destination derived; explicit `true` retained |
|
||||||
|
| omitted | `false` | destination derived; explicit `false` retained |
|
||||||
|
| explicit | either value | explicit destination and requiredness retained |
|
||||||
|
|
||||||
|
`narratio config validate`, `narratio config show`, and ordinary pipeline
|
||||||
|
loading accept the corrected cases without an unrecorded-change diagnostic.
|
||||||
|
No operator configuration migration or workaround is required.
|
||||||
|
|
||||||
|
## Release
|
||||||
|
|
||||||
|
This is a backward-compatible correction to documented configuration behavior
|
||||||
|
and should be released as Narratio `v2.1.1`. The release note should identify
|
||||||
|
the corrected explicit-required/derived-destination interaction and state that
|
||||||
|
no special upgrade action is required.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- The focused regression test fails against v2.1.0 behavior and passes after
|
||||||
|
the fix.
|
||||||
|
- Publish rules with explicit `required: true` or `required: false` and omitted
|
||||||
|
derivable destinations load and validate successfully.
|
||||||
|
- Derived destinations and explicit requiredness appear correctly in the safe
|
||||||
|
effective configuration.
|
||||||
|
- Existing destination derivation, publish validation, imports, profiles, and
|
||||||
|
configuration inspection tests remain green.
|
||||||
|
- The complete repository validation and release-candidate checks pass.
|
||||||
|
- The correction is published as the immutable `v2.1.1` tag with the required
|
||||||
|
release note.
|
||||||
431
docs/roadmap/implementation.md
Normal file
431
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,431 @@
|
|||||||
|
# Implementation Plan: Speaker Checkpoints and Replayable Uploads
|
||||||
|
|
||||||
|
Status: Implementation complete; Stages 1-7 complete.
|
||||||
|
|
||||||
|
## Scope and Instructions
|
||||||
|
|
||||||
|
Implement [the feature roadmap](transcribe-checkpoints-and-replay.md). This plan
|
||||||
|
replaces the completed Notarius-profile plan; that work remains in Git history.
|
||||||
|
Complete the numbered stages in order, one bounded implementation prompt per
|
||||||
|
stage for gpt-5.6-terra. Record completion and actual verification results here.
|
||||||
|
Intermediate stages are not a finished feature and must not be released alone.
|
||||||
|
|
||||||
|
Before working, read `AGENTS.md`, `docs/development.md`, all three policies in
|
||||||
|
`docs/policy/`, the roadmap, and the stage-specific preparation below. Use the
|
||||||
|
maintained Go 1.26 toolchain. Preserve operator choices, speaker filenames,
|
||||||
|
bounded concurrency, streaming, and existing credentials/settings. Do not add
|
||||||
|
new commands, dependencies, service requirements, arbitrary speaker limits,
|
||||||
|
strict transcript schemas, or a generic checkpoint framework. No release or
|
||||||
|
upstream service work is part of this plan.
|
||||||
|
|
||||||
|
## Settled Design Decisions
|
||||||
|
|
||||||
|
### Progress Representation and Recovery
|
||||||
|
|
||||||
|
Add optional `transcribe_state_version` (version 1) and `transcribe_speakers`
|
||||||
|
to `manifest.StageRecord`. The latter is a map keyed by the existing derived
|
||||||
|
speaker ID, containing completed results only. Absence means that work must be
|
||||||
|
done; a separate pending/running/failed state machine is unnecessary. Aggregate
|
||||||
|
stage status/error remains authoritative for invocation failure. A versioned
|
||||||
|
empty collection is meaningful: no earlier completions are reusable.
|
||||||
|
|
||||||
|
Use a typed `TranscribeSpeakerRecord` with fingerprint version/digest,
|
||||||
|
session-relative audio path, audio SHA-256 and size, one `ArtifactRecord` output
|
||||||
|
of kind `transcript_raw`, output size, and producer run ID. The output carries
|
||||||
|
its session-relative canonical path and SHA-256. No new transcript source IDs,
|
||||||
|
synthetic schema contracts, timestamps, or historical attempt collections are
|
||||||
|
needed. Existing aggregate diagnostics can retain attempt/status/duration data
|
||||||
|
for work performed in the current invocation; do not fabricate that information
|
||||||
|
for reused outputs.
|
||||||
|
|
||||||
|
Keep the session-manifest schema identifier unchanged for this optional additive
|
||||||
|
state; validate supported per-stage versions and retain strict decoding. New
|
||||||
|
Narratio reads old manifests. Old binaries may reject the added fields under
|
||||||
|
their existing strict decoding; document this downgrade limitation.
|
||||||
|
|
||||||
|
For pre-feature manifests with no speaker evidence, regenerate transcription
|
||||||
|
once when transcribe is next selected, even if the aggregate status succeeded.
|
||||||
|
Old records do not bind individual transcript results to the audio bytes they
|
||||||
|
used. Do not guess that association from current files, old `per_file` metadata,
|
||||||
|
or abandoned run directories. This is the conservative upgrade path, not a new
|
||||||
|
general migration system. No other stage or unselected range is forcibly run.
|
||||||
|
|
||||||
|
### Reuse and Persistence
|
||||||
|
|
||||||
|
Compute a versioned per-speaker fingerprint from speaker ID, session-relative
|
||||||
|
audio identity, actual audio SHA-256/size, and the existing transcribe semantic
|
||||||
|
configuration fingerprint (URL and language). Do not include invocation ID,
|
||||||
|
absolute workspace root, concurrency, retry settings, timeout, or credentials.
|
||||||
|
Stream input hashing with cancellation; do not trust stored input checksums
|
||||||
|
alone. Recheck input bytes before accepting a new result to catch changes
|
||||||
|
observed during transcription; fail that job without checkpointing mismatched
|
||||||
|
evidence. This is not a hostile-filesystem snapshot or concurrent-editor
|
||||||
|
transaction guarantee and requires no extra audio copies or file locks.
|
||||||
|
|
||||||
|
Validate reusable output identity, canonical location, JSON, checksum, and size.
|
||||||
|
Use existing confinement/file primitives. Missing or corrupt regular outputs
|
||||||
|
become work; unsafe paths/non-regular conflicts follow existing safety errors,
|
||||||
|
not recursive deletion. Missing required audio is an input error, not permission
|
||||||
|
to reuse an unverifiable transcript. Removed inputs are removed from the current
|
||||||
|
collection and aggregate outputs without deleting arbitrary files.
|
||||||
|
|
||||||
|
One stage-owned mutex serializes candidate collection construction, manifest
|
||||||
|
save, and installation of committed state. Save a detached candidate and only
|
||||||
|
then update live authority, following analyze's pattern. Workers must not mutate
|
||||||
|
the live manifest outside that serialized section. Keep expensive HTTP work
|
||||||
|
outside the mutex. Stop dispatch/acceptance on checkpoint failure, join workers
|
||||||
|
before return, and preserve the last successfully committed collection.
|
||||||
|
|
||||||
|
Before launching requests, durably checkpoint the reconciled collection. For
|
||||||
|
force, that collection is empty: this prevents later ordinary retries from
|
||||||
|
resurrecting pre-force outputs. If this initial save fails, do not start requests
|
||||||
|
or overwrite outputs. Force again clears the current collection again; ordinary
|
||||||
|
retry retains replacements already checkpointed after the previous force.
|
||||||
|
|
||||||
|
Save each validated result at its canonical raw-transcript location before
|
||||||
|
checkpointing it. Run-local files remain staging/diagnostics, not reuse
|
||||||
|
dependencies. Bytes left by an unsuccessful checkpoint have no completion
|
||||||
|
authority. A worker may checkpoint a completed result using the parent stage
|
||||||
|
context even if a sibling canceled the work context; parent cancellation or a
|
||||||
|
checkpoint failure may prevent that save. Guarantee retention of committed
|
||||||
|
work, not all responses that happened to finish during cancellation.
|
||||||
|
|
||||||
|
### Multipart Replay Mechanism
|
||||||
|
|
||||||
|
Replace the pipe/producer upload mechanism with a small pull-based multipart
|
||||||
|
body backed by the audio reader. Use `mime/multipart.Writer` to encode a bounded
|
||||||
|
metadata prefix and suffix around the file payload: capture the prefix after
|
||||||
|
`CreateFormFile`, then encode the language field and closing boundary as the
|
||||||
|
suffix. Generate this framing once per application attempt. Each body combines
|
||||||
|
fresh readers for prefix, freshly opened audio, and suffix using `io.MultiReader`.
|
||||||
|
No full-audio buffer or temporary multipart spool is needed.
|
||||||
|
|
||||||
|
The initial body and `Request.GetBody` use the same factory/framing. Each owns
|
||||||
|
an independent audio handle. Give bodies idempotent Close and context-triggered
|
||||||
|
closure (for example `context.AfterFunc`), without holding a mutex across a
|
||||||
|
blocking Read. Read/Close coordination must release a blocked source and retain
|
||||||
|
genuine source-read/close errors while not treating deliberate early closure as
|
||||||
|
source failure. The attempt owns cleanup of all created bodies, including
|
||||||
|
replays; reject new body creation after its context expires.
|
||||||
|
|
||||||
|
Remove the old producer `Wait` dependency. Track body-local source errors with
|
||||||
|
synchronization so a swallowed upload read error cannot become a successful
|
||||||
|
transcript, and a deliberately closed original body cannot poison a successful
|
||||||
|
replay. Keep the outer retry policy and per-attempt context. Do not add replay
|
||||||
|
counters to the user retry budget or depend on Go's exact internal retry count.
|
||||||
|
Preserve ordinary HTTP redirect policy; body replay can also enable Go's
|
||||||
|
body-preserving redirects. Do not introduce a new redirect restriction.
|
||||||
|
|
||||||
|
## Stage 1: Add Durable Speaker Completion State
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Added optional versioned transcribe-owned
|
||||||
|
per-speaker completion records, strict load/save validation, isolated collection
|
||||||
|
cloning, legacy aggregate-only compatibility, and lifecycle-preservation
|
||||||
|
coverage. Verified with `go test ./internal/manifest` and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/manifest/manifest.go`, `store.go`, `analyze_state.go`,
|
||||||
|
their tests, and `docs/internal/manifest.md`.
|
||||||
|
|
||||||
|
1. Implement the additive fields, record type, version constants, validation,
|
||||||
|
and deep-copy helper described above in a focused transcribe state file.
|
||||||
|
2. Validate ownership by the transcribe stage, supported versions, required
|
||||||
|
completed-record evidence, output kind, safe relative paths, SHA-256 values,
|
||||||
|
sizes, and producer identity consistency. Use existing speaker IDs as-is;
|
||||||
|
do not apply analysis-artifact key regexes or copy unrelated analysis limits.
|
||||||
|
3. Integrate validation with both load and save. Preserve speaker state through
|
||||||
|
running/failed/stale transitions and aggregate result clearing. Generic stage
|
||||||
|
invalidation does not discard checkpoints; reconciliation checks their data.
|
||||||
|
4. Test round-trip populated and versioned-empty collections, old manifests
|
||||||
|
without fields, malformed evidence, deep-copy isolation, and lifecycle
|
||||||
|
preservation. Avoid imposing JSON formatting or map order as contracts.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/manifest`.
|
||||||
|
|
||||||
|
Done when durable storage can represent partial completion independently of
|
||||||
|
aggregate status, without activating new stage behavior yet.
|
||||||
|
|
||||||
|
## Stage 2: Implement Read-Only Speaker Reconciliation
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Extracted shared deterministic transcribe job
|
||||||
|
construction and added read-only, cancellation-aware per-speaker input and
|
||||||
|
output reconciliation. Covered unchanged reuse, semantic and operational
|
||||||
|
settings, changed/added/removed inputs, legacy and forced work, missing/corrupt
|
||||||
|
outputs, unsafe evidence, and missing audio without connecting runtime skip or
|
||||||
|
checkpoint behavior. Verified with
|
||||||
|
`go test ./internal/stage ./internal/manifest` and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/stage/transcribe.go`, `semantic_contracts_initial.go`,
|
||||||
|
`analyze_reconciliation.go`, `extract_resume.go`, their tests, and the artifact
|
||||||
|
path/integrity helpers they use.
|
||||||
|
|
||||||
|
1. Extract the existing audio discovery/job construction into a shared,
|
||||||
|
transcribe-local reconciliation path. Preserve current input selection and
|
||||||
|
speaker identity behavior. Build sorted jobs and classify completed results
|
||||||
|
against the current input set and evidence defined above.
|
||||||
|
2. Add pure fingerprint construction and streamed input/output inspection.
|
||||||
|
Resolve manifest input paths with existing session-relative/absolute reading
|
||||||
|
helpers; do not change input precedence or assume the current working directory
|
||||||
|
is the session root. Newly written checkpoint paths are always relative.
|
||||||
|
3. Return current records and work-to-do without writing files or manifest state.
|
||||||
|
Force selects all work and an empty starting collection; old unversioned
|
||||||
|
transcribe state supplies no reusable records. Do not connect aggregate skip
|
||||||
|
validation until execution can produce the new evidence in Stage 3.
|
||||||
|
4. Test unchanged reuse; one changed audio among unchanged speakers; URL/language
|
||||||
|
changes; operational-setting exclusions; added/removed speakers; missing and
|
||||||
|
corrupt regular outputs; unsafe record paths; missing audio; and force.
|
||||||
|
Prefer compact tables and real temporary files over mocked hashing.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/stage ./internal/manifest`.
|
||||||
|
|
||||||
|
Done when one read-only decision path describes exactly which speakers need
|
||||||
|
work, without altering existing runtime behavior prematurely.
|
||||||
|
|
||||||
|
## Stage 3: Execute and Checkpoint Speakers Incrementally
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Connected reconciliation to bounded execution,
|
||||||
|
durably checkpointed the starting and per-speaker collections through detached
|
||||||
|
manifest candidates, preserved committed work across failures and cancellation,
|
||||||
|
rebuilt deterministic aggregate outputs from reusable evidence, and added
|
||||||
|
speaker-aware resume validation. Updated production-style fixtures and covered
|
||||||
|
serialized concurrent saves, force/retry replacement, input rechecks, save
|
||||||
|
failures, partial recovery, and aggregate reconstruction. Verified with
|
||||||
|
`go test ./internal/stage ./internal/app`, `go test -race ./internal/stage`, and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/stage/transcribe.go`, `analyze_execution_state.go`,
|
||||||
|
`run_local.go`,
|
||||||
|
`internal/app/runner.go`, and transcribe/checkpoint test fixtures.
|
||||||
|
|
||||||
|
1. Wire reconciliation into Run, require the existing ManifestStore collaborator,
|
||||||
|
and checkpoint the starting collection before launching HTTP work. Implement
|
||||||
|
detached candidate saves and serialized live-state updates. Audit Store.Save's
|
||||||
|
normalization side effects so a failed candidate save cannot mutate shared
|
||||||
|
authority; use detached records/collections wherever it may write.
|
||||||
|
2. Retain bounded errgroup execution and fail-fast cancellation. Check the work
|
||||||
|
context before beginning each queued request. After success, validate adapter
|
||||||
|
output path/JSON, recheck audio evidence, materialize canonical bytes, compute
|
||||||
|
output evidence, and checkpoint the new record. Check a checkpoint-failure
|
||||||
|
latch under the serialization lock before accepting further completions.
|
||||||
|
3. Preserve committed successes on all error/cancellation exits, including
|
||||||
|
starting-save, canonical-write, per-speaker-save, and aggregate-save failures.
|
||||||
|
Return meaningful speaker/error context and let the runner retain its existing
|
||||||
|
terminal failure behavior. Do not report partial aggregate outputs as success.
|
||||||
|
4. Assemble final outputs in deterministic speaker order from the reconciled
|
||||||
|
completed records, including reused speakers, with original producer IDs and
|
||||||
|
checksums. If all records are current but aggregate status is failed/running,
|
||||||
|
finish without WhisperX calls and rebuild the aggregate success.
|
||||||
|
5. Implement `transcribeStage.ValidateResume` using the same read-only
|
||||||
|
reconciliation, so a succeeded stage cannot skip changed audio or invalid
|
||||||
|
transcripts. Retain the existing URL/language semantic check and runner
|
||||||
|
invalidation behavior; no whole-config digest invalidation.
|
||||||
|
6. Adapt existing tests that require no canonical partial files: the new contract
|
||||||
|
permits checkpointed files but forbids aggregate success. Update fixtures to
|
||||||
|
provide real manifest stores and records rather than a production fallback
|
||||||
|
that silently disables checkpointing.
|
||||||
|
7. Test A/B completion followed by C failure and D unattempted (concurrency 1),
|
||||||
|
reload and retry only C/D; concurrently finishing speakers without lost saves;
|
||||||
|
failure and cancellation after a checkpoint; save failure retaining earlier
|
||||||
|
authority; and force failure followed by ordinary retry without old-result reuse.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/stage ./internal/app` and
|
||||||
|
`go test -race ./internal/stage`.
|
||||||
|
|
||||||
|
Done when incremental progress and aggregate success work together safely under
|
||||||
|
ordinary, concurrent, canceled, and forced execution.
|
||||||
|
|
||||||
|
## Stage 4: Verify Runner, Merge, and Durable Lifecycle Integration
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Added real-persistence application coverage for
|
||||||
|
cross-invocation retry, interrupted running state, failed aggregate saves,
|
||||||
|
force/reset persistence, downstream invalidation, and one-time legacy upgrade.
|
||||||
|
Versioned merge input discovery now requires aggregate success and an exact
|
||||||
|
match between current speaker evidence and declared aggregate outputs, while
|
||||||
|
legacy/manual directory fallback remains available. Restore coverage preserves
|
||||||
|
session-relative speaker paths, leaves unpublished raw transcripts absent, and
|
||||||
|
canonical transcripts remain reusable after run-directory cleanup. Verified
|
||||||
|
with `go test ./internal/app ./internal/stage ./internal/manifest`,
|
||||||
|
`go test -race ./internal/app`, and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/app/runner.go`, `runner_runtime.go`,
|
||||||
|
`analyze_checkpoint_recovery_test.go`, `runner_terminal_failure_test.go`,
|
||||||
|
`restore_manifest_paths.go`, `internal/stage/merge.go`, and existing publish,
|
||||||
|
restore, and cleanup tests routed from `docs/internal/overview.md`.
|
||||||
|
|
||||||
|
1. Add application-level tests with real manifest persistence and fake WhisperX:
|
||||||
|
failed stage across separate invocations; all speakers committed before a
|
||||||
|
failed aggregate save; interrupted running state; force/reset persistence;
|
||||||
|
and unchanged upstream/succeeded downstream invalidation behavior. Inject
|
||||||
|
errors at durable boundaries, not every private method call.
|
||||||
|
2. Close `discoverRawTranscripts`' directory-scan fallback for versioned
|
||||||
|
transcribe state: incomplete aggregate state is not usable, and completed
|
||||||
|
state consumes only declared current aggregate outputs. Keep legacy/manual
|
||||||
|
fallback behavior for manifests without the new state rather than banning
|
||||||
|
previously allowed standalone workflows. Test that partial files and removed
|
||||||
|
speaker files cannot leak into a versioned merge, including direct stage calls.
|
||||||
|
3. Audit manifest copies/projections, remote manifest publication, restored path
|
||||||
|
handling, and cleanup. Preserve relative speaker paths on restore and verify
|
||||||
|
they resolve under the destination session, not the producer machine. Reuse
|
||||||
|
existing confinement validation, not a second path-policy layer.
|
||||||
|
4. Add focused round-trip coverage through existing publish/restore fixtures for
|
||||||
|
speaker records. Restored outputs absent from the published selection are
|
||||||
|
non-reusable locally; do not silently expand remote publication to every raw
|
||||||
|
speaker or upload audio as a new default. Removing run directories must not
|
||||||
|
remove canonical checkpointed transcripts or make reuse require run files.
|
||||||
|
5. Verify old succeeded manifests are readable and rerun once when transcribe is
|
||||||
|
selected, then reuse normally. Keep old manifests usable for other commands
|
||||||
|
and bounded ranges. Do not automatically import old unrecorded JSON files.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/app ./internal/stage ./internal/manifest` and
|
||||||
|
`go test -race ./internal/app`.
|
||||||
|
|
||||||
|
Done when the new authority survives runner and storage lifecycles and cannot
|
||||||
|
expose incomplete transcripts downstream.
|
||||||
|
|
||||||
|
## Stage 5: Implement Replayable Streaming Multipart Bodies
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Replaced the producer-goroutine/pipe upload
|
||||||
|
with bounded standard-library multipart framing and independently opened,
|
||||||
|
pull-based streaming bodies shared by the initial request and `GetBody`.
|
||||||
|
Bodies are tracked, idempotently closed, canceled with their attempt context,
|
||||||
|
and retain genuine read/close failures without treating an intentionally
|
||||||
|
abandoned original as a replay failure. Added focused coverage for identical
|
||||||
|
framing, partial-read restart, simultaneous handles, source failures, and
|
||||||
|
blocked-reader release. Verified with `go test ./internal/adapters/whisperx`,
|
||||||
|
`go test -race ./internal/adapters/whisperx` (including ten repeated runs), and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/adapters/whisperx/http.go`, `http_test.go`,
|
||||||
|
`docs/integrations/whisperx.md`, and the installed Go net/http Request/GetBody
|
||||||
|
contract. Keep this change inside the WhisperX adapter.
|
||||||
|
|
||||||
|
1. Implement the pull-based framing/body factory described above, using standard
|
||||||
|
multipart encoding rather than hand-escaped MIME headers. Add no new payload
|
||||||
|
cap or upload-size setting. Preserve file-before-language wire semantics and
|
||||||
|
the existing form names.
|
||||||
|
2. Install the initial body and GetBody with the same per-attempt boundary.
|
||||||
|
Track and close all opened bodies even if a custom transport returns early;
|
||||||
|
make cleanup/context callbacks safe against overlapping reads and closes.
|
||||||
|
A GetBody failure must be returned, not converted into an empty upload.
|
||||||
|
3. Replace pipe-specific Wait/error paths in doTranscribeAttempt with source-error
|
||||||
|
handling appropriate to pull bodies. Genuine source errors remain visible;
|
||||||
|
intentional closure during transport retry is not itself a source error.
|
||||||
|
Keep response bounds, JSON validation, output installation, retry
|
||||||
|
classification/delay, cancellation, and application attempt reporting intact.
|
||||||
|
4. Remove obsolete producer-only helpers rather than retaining two upload paths.
|
||||||
|
Adapt tests tied to producer goroutines to assert streaming and resource
|
||||||
|
release, which are the actual contracts.
|
||||||
|
5. Add focused body tests: original/replayed bytes parse identically; replay
|
||||||
|
after partial consumption starts from the beginning; simultaneous bodies use
|
||||||
|
independent handles; source failures propagate; Close/cancel releases a
|
||||||
|
blocked reader without deadlock. Use synthetic audio and gated readers.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/adapters/whisperx` and
|
||||||
|
`go test -race ./internal/adapters/whisperx`.
|
||||||
|
|
||||||
|
Done when replay is available without whole-file buffering or producer-lifetime
|
||||||
|
ambiguity, and all existing HTTP behavior tests still pass.
|
||||||
|
|
||||||
|
## Stage 6: Verify Replay Through the HTTP Adapter Boundary
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Added a deterministic transport regression
|
||||||
|
that abandons a partially consumed original request, obtains and parses a fresh
|
||||||
|
multipart replay from `GetBody`, and installs the returned transcript within one
|
||||||
|
application attempt. Covered replay open/read failures, early responses,
|
||||||
|
parent cancellation during replay, and disposal of every opened source, while
|
||||||
|
proving an abandoned original does not poison a valid replay. Added a real HTTP
|
||||||
|
retry-exhaustion integration that durably retains an earlier speaker and resumes
|
||||||
|
only the uncommitted speakers. This controlled boundary verifies Narratio's
|
||||||
|
replay contract, not recovery from every real proxy protocol failure. Verified
|
||||||
|
with `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`,
|
||||||
|
`go test -race ./internal/adapters/whisperx ./internal/app ./internal/stage`,
|
||||||
|
ten repeated race-enabled adapter runs, and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: Existing injectable HTTPClient/RoundTripper seam and local-server
|
||||||
|
tests in `internal/adapters/whisperx/http_test.go`; app WhisperX wiring tests.
|
||||||
|
|
||||||
|
1. Add a deterministic transport-boundary regression: consume part of the
|
||||||
|
original request, close it as a transport would, call GetBody, consume and
|
||||||
|
parse the replay using the request Content-Type, then return valid JSON.
|
||||||
|
Verify exact audio/language, installed output, and one application attempt.
|
||||||
|
This must fail before replay support; merely checking non-nil GetBody is
|
||||||
|
insufficient. Do not assert Go's internal retry count or require a flaky
|
||||||
|
handcrafted HTTP/2 peer in the default suite.
|
||||||
|
2. Cover replay-open/read failure, early response, parent cancellation during
|
||||||
|
replay, and disposal of all body handles. Reuse Stage 5 body coverage rather
|
||||||
|
than repeat every low-level case. Prove an intentionally abandoned original
|
||||||
|
body does not turn a valid replay response into failure.
|
||||||
|
3. Retain local HTTP tests for application retries, non-retryable responses,
|
||||||
|
response limits, invalid JSON, and bounded streaming; exercise that an
|
||||||
|
exhausted HTTP failure still leaves earlier speaker checkpoints reusable
|
||||||
|
using the existing stage/app seams.
|
||||||
|
4. Record the verification boundary honestly: controlled transport replay proves
|
||||||
|
Narratio's body contract, not recovery from every real proxy PROTOCOL_ERROR.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`
|
||||||
|
and race tests for changed packages.
|
||||||
|
|
||||||
|
Done when the replay defect is protected at the adapter boundary and the
|
||||||
|
application retry/checkpoint behavior remains intact.
|
||||||
|
|
||||||
|
## Stage 7: Document and Validate the Completed Feature
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Updated the canonical transcribe, merge,
|
||||||
|
manifest, WhisperX, operations, and troubleshooting documentation for durable
|
||||||
|
speaker reconciliation, serialized checkpoints, aggregate gating, durable
|
||||||
|
force reset, legacy upgrade and downgrade limits, replayable streaming bodies,
|
||||||
|
resource ownership, and the absence of an exactly-once remote guarantee.
|
||||||
|
Reviewed the completed implementation against the feature roadmap and
|
||||||
|
implementation-discipline guidance. Verified with `go test ./...`,
|
||||||
|
`go test -race ./...`, `go vet ./...`, `go build ./...`,
|
||||||
|
`go test ./internal/doccheck`,
|
||||||
|
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
1. Update canonical transcribe and manifest internals for the collection,
|
||||||
|
reconciliation, durable-reset force behavior, serialized saves, and aggregate
|
||||||
|
gate. Remove obsolete all-speakers-before-canonical-write descriptions.
|
||||||
|
2. Update the WhisperX contract for replayable bounded streaming, application
|
||||||
|
attempts versus transport replay, resource ownership, and the lack of an
|
||||||
|
exactly-once server guarantee. Do not claim the remote protocol issue is fixed.
|
||||||
|
3. Update operations/troubleshooting with ordinary resume versus force, durable
|
||||||
|
partial success, old-manifest one-time retranscription on selecting transcribe,
|
||||||
|
downgrade limitations, and private-model-change guidance. No new CLI flags
|
||||||
|
or promises of automatic historical-file recovery. Keep detailed wire and
|
||||||
|
storage contracts in their canonical owners.
|
||||||
|
4. Review changes against the feature roadmap and the implementation-discipline
|
||||||
|
guidance. Do not elevate selected storage encoding, internal ordering, test
|
||||||
|
synchronization, or exact error wording into new product guarantees.
|
||||||
|
5. Run the full repository gates below; document results and update completion
|
||||||
|
statuses only after they pass. Leave release preparation for a separate task.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./...
|
||||||
|
go test ./internal/doccheck
|
||||||
|
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None requiring user input. The agreed roadmap supplies the product decisions;
|
||||||
|
this plan selects a completed-only manifest collection, conservative old-record
|
||||||
|
handling, and pull-based replay as implementation choices. No stronger remote
|
||||||
|
reliability guarantee, new operator restriction, or generic orchestration
|
||||||
|
surface is proposed.
|
||||||
151
docs/roadmap/notarius-config-profile.md
Normal file
151
docs/roadmap/notarius-config-profile.md
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
# Notarius Configuration Profile Selection
|
||||||
|
|
||||||
|
Status: Implemented.
|
||||||
|
|
||||||
|
## Goal and Authority
|
||||||
|
|
||||||
|
Implement the consumer behavior in the
|
||||||
|
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
|
||||||
|
configuration overlay explicitly select a Notarius configuration overlay.
|
||||||
|
Operators can select testing once in Narratio while each application retains
|
||||||
|
ownership of its own model choices and other settings.
|
||||||
|
|
||||||
|
This roadmap describes the target state and required change areas, not an
|
||||||
|
implementation sequence; the [implementation plan](implementation.md) owns
|
||||||
|
the ordered work and implementation decisions. The feature request is the scope
|
||||||
|
authority; the mechanisms suggested below do not establish additional
|
||||||
|
user-facing contracts.
|
||||||
|
|
||||||
|
## Underlying Issue
|
||||||
|
|
||||||
|
Narratio can select its own testing profile, but extraction currently invokes
|
||||||
|
Notarius without a configuration-profile selector. Consequently, a Narratio
|
||||||
|
testing invocation cannot explicitly select the independently deployed Notarius
|
||||||
|
testing overlay through pipeline configuration.
|
||||||
|
|
||||||
|
Adding individual downstream model or tuning fields would duplicate Notarius's
|
||||||
|
configuration responsibilities. Passing its existing profile selector provides
|
||||||
|
the requested flexibility without that duplication.
|
||||||
|
|
||||||
|
## Targeted End State
|
||||||
|
|
||||||
|
### Explicit Configuration Composition
|
||||||
|
|
||||||
|
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
|
||||||
|
name, usable in Narratio's base configuration and ordinary overlays. For
|
||||||
|
example, Narratio's `profiles/testing.yml` may contain:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: testing
|
||||||
|
```
|
||||||
|
|
||||||
|
Selecting Narratio's `--profile testing` then selects that explicit downstream
|
||||||
|
profile whenever extraction executes. The two profile names need not match.
|
||||||
|
Narratio does not automatically forward its own profile name.
|
||||||
|
|
||||||
|
When supplied, the selection is passed through Notarius's `--config-profile`
|
||||||
|
argument. Omitted or null values add no profile-selection flag and preserve
|
||||||
|
existing downstream selection behavior. Notarius's selector semantics remain
|
||||||
|
authoritative, including empty and `default` selections for the composed base
|
||||||
|
configuration. Narratio does not introduce a stricter profile-name policy.
|
||||||
|
|
||||||
|
This field selects a ConfigKit configuration overlay, not a PromptKit profile
|
||||||
|
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
|
||||||
|
selection. Notarius resolves overlays relative to its own configuration root;
|
||||||
|
Narratio neither resolves those files against its own root nor rewrites them.
|
||||||
|
|
||||||
|
### Execution, Reuse, and Failure
|
||||||
|
|
||||||
|
A changed downstream profile selection must not silently reuse Narratio's
|
||||||
|
completed extraction from the previous selection. Existing dependent-stage
|
||||||
|
invalidation applies when extraction changes; the feature does not change the
|
||||||
|
pipeline order or transcript-generation behavior.
|
||||||
|
|
||||||
|
Notarius continues to own downstream overlay discovery, composition, validation,
|
||||||
|
and cache compatibility. A missing or invalid selected overlay must fail
|
||||||
|
extraction through the existing subprocess error path, without silently falling
|
||||||
|
back to production or reporting successful extraction.
|
||||||
|
|
||||||
|
Existing resume and chunk-cache controls, including force precedence, remain
|
||||||
|
in place. Profile selection does not itself mean forced regeneration or cache
|
||||||
|
deletion. Narratio's decision to execute extraction and Notarius's decision to
|
||||||
|
reuse compatible work remain separate responsibilities.
|
||||||
|
|
||||||
|
Changes to private Notarius configuration behind an unchanged selector retain
|
||||||
|
the existing operator responsibility to force extraction when needed. There is
|
||||||
|
no new downstream file inspection or content-tracking requirement.
|
||||||
|
|
||||||
|
Credential forwarding remains independently configured through
|
||||||
|
`notarius.llm_api_key`. A Narratio testing overlay may select a different
|
||||||
|
credential reference alongside the downstream profile; no new credential
|
||||||
|
discovery or provider-name mapping is introduced.
|
||||||
|
|
||||||
|
## Required Change Areas
|
||||||
|
|
||||||
|
### Configuration and Inspection
|
||||||
|
|
||||||
|
Extend the typed Notarius configuration and its existing safe effective-YAML
|
||||||
|
projection to carry the optional field. Preserve the ordinary configuration
|
||||||
|
composition behavior for base values, overlay overrides, and null values.
|
||||||
|
Configuration inspection must reflect the selected field without exposing
|
||||||
|
credential values. No replacement configuration loader or downstream preflight
|
||||||
|
loader is needed.
|
||||||
|
|
||||||
|
### Stage and Adapter Boundary
|
||||||
|
|
||||||
|
Carry the selection from resolved configuration through the extraction request
|
||||||
|
to the subprocess adapter. Construct the optional CLI flag using the existing
|
||||||
|
argument-vector mechanism, alongside the current configuration path, references,
|
||||||
|
credential forwarding, and cache controls. Retain normal Notarius error handling.
|
||||||
|
|
||||||
|
### Extraction Reuse Evidence
|
||||||
|
|
||||||
|
Account for the downstream selection in the existing extraction reuse evidence.
|
||||||
|
The current semantic-configuration and invocation fingerprints are the relevant
|
||||||
|
change points. Keep their treatment of selection consistent so that a profile
|
||||||
|
change cannot bypass the required rerun and downstream invalidation.
|
||||||
|
|
||||||
|
This roadmap does not require a new manifest schema, separate profile-indexed
|
||||||
|
workspaces, historical per-profile result retention, or an upgrade-wide forced
|
||||||
|
rerun.
|
||||||
|
|
||||||
|
### Verification and Documentation
|
||||||
|
|
||||||
|
Use focused behavioral coverage for configuration composition and inspection,
|
||||||
|
explicit argument forwarding and omission, changed-selection reuse decisions,
|
||||||
|
and ordinary subprocess failure handling. Reuse existing test coverage where
|
||||||
|
it already protects unchanged credentials, force behavior, and invalidation;
|
||||||
|
do not duplicate Notarius's complete profile-validation or cache test suites.
|
||||||
|
|
||||||
|
When implemented, update the canonical
|
||||||
|
[configuration reference](../config.md),
|
||||||
|
[Notarius integration contract](../integrations/notarius.md),
|
||||||
|
[extract internals](../internal/stage-extract.md), and relevant maintained
|
||||||
|
examples. The development workflow in [Operations](../operations.md) should
|
||||||
|
explain explicit profile composition and the existing need to force extraction
|
||||||
|
after private downstream edits. Clarify that profile selection does not create
|
||||||
|
an isolated testing workspace or separate current extraction state.
|
||||||
|
|
||||||
|
## Compatibility and Scope Boundaries
|
||||||
|
|
||||||
|
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
|
||||||
|
required. Document the capability needed when the option is used, without adding
|
||||||
|
an unconditional runtime version gate for existing unconfigured deployments.
|
||||||
|
|
||||||
|
This feature adds no Narratio CLI selector, automatic profile propagation,
|
||||||
|
Notarius LLM-profile selector, downstream configuration rewriting, new
|
||||||
|
credential mechanism, or generalized recursive orchestration. Existing output
|
||||||
|
and receipt contracts remain unchanged.
|
||||||
|
|
||||||
|
The consumer's Ansible role and suite updates are follow-up work outside this
|
||||||
|
repository. They can deploy downstream overlays and populate this field after
|
||||||
|
the interface is released.
|
||||||
|
|
||||||
|
## Scope Review
|
||||||
|
|
||||||
|
No stronger requirements or new invariants are proposed beyond the feature
|
||||||
|
request and existing repository contracts. In particular, minimizing one-time
|
||||||
|
upgrade invalidation and avoiding reruns between equivalent base selectors are
|
||||||
|
implementation considerations, not additional acceptance guarantees imposed
|
||||||
|
by this roadmap. No user decisions are currently required before planning.
|
||||||
104
docs/roadmap/remediation.md
Normal file
104
docs/roadmap/remediation.md
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
# Transcription Checkpoint Review Remediation
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
|
||||||
|
upload-replay roadmap and the original user request. No production changes were
|
||||||
|
made during review. The two failures below were reproduced with temporary tests
|
||||||
|
using the existing stage fixtures; those temporary tests were removed afterward.
|
||||||
|
|
||||||
|
## 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
|
||||||
|
|
||||||
|
Priority: P2. Status: Complete.
|
||||||
|
|
||||||
|
Implemented read-only aggregate identity/evidence checks before resume skips.
|
||||||
|
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
|
||||||
|
repair without HTTP requests, downstream invalidation, and exclusion of removed
|
||||||
|
speakers from merge without deleting their files. Equivalent path forms and
|
||||||
|
output ordering remain reusable. Focused app/stage tests and documentation
|
||||||
|
checks pass.
|
||||||
|
|
||||||
|
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
|
||||||
|
consumed there.
|
||||||
|
|
||||||
|
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
|
||||||
|
only establishes that the remaining speakers need no transcription; it does
|
||||||
|
not establish that the stored speaker collection and aggregate outputs match
|
||||||
|
the current input set. Removing a speaker while the remaining inputs are
|
||||||
|
unchanged produces an empty work list, so a succeeded record can be skipped
|
||||||
|
without pruning the removed speaker. Its old aggregate output can then remain
|
||||||
|
available to merge. Likewise, missing/inconsistent aggregate output references
|
||||||
|
are not repaired when all individual checkpoints remain valid. A prepare rerun
|
||||||
|
may already stale transcribe in some workflows, but the resume validator cannot
|
||||||
|
rely on that to satisfy its own current-input/output checks.
|
||||||
|
|
||||||
|
Reproduction: create complete alice/bob checkpoint records and aggregate
|
||||||
|
outputs with the existing fixture, mark the stage succeeded, remove bob from
|
||||||
|
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
|
||||||
|
despite still retaining bob in the stored collection and aggregate outputs.
|
||||||
|
|
||||||
|
Required change:
|
||||||
|
|
||||||
|
- Distinguish no HTTP work from no reconciliation work. Validate that the
|
||||||
|
committed speaker set and aggregate transcript references represent the
|
||||||
|
reconciled current set, including relevant identity/evidence, before skipping.
|
||||||
|
- When only collection/output repair is needed, return non-resumable and let
|
||||||
|
normal execution checkpoint the pruned collection and rebuild aggregate
|
||||||
|
outputs without retranscribing reusable speakers. Preserve existing dependent
|
||||||
|
invalidation; do not force otherwise-current speaker jobs.
|
||||||
|
- Keep resume inspection read-only. Do not fix this by deleting incidental
|
||||||
|
transcript files or changing stage selection rules.
|
||||||
|
|
||||||
|
Verification: add removed-only input coverage (not just removal combined with
|
||||||
|
an added speaker), plus incomplete/inconsistent aggregate output coverage.
|
||||||
|
Exercise the runner skip path and assert zero additional WhisperX requests,
|
||||||
|
correct rebuilt outputs, and no removed speaker reaching merge.
|
||||||
|
|
||||||
|
## 2. Remove the Unrequested Positive Audio-Size Requirement
|
||||||
|
|
||||||
|
Priority: P2. Status: Complete.
|
||||||
|
|
||||||
|
Audio evidence now permits zero size while still rejecting negative values.
|
||||||
|
Regression tests verify zero-size manifest round trips, successful service
|
||||||
|
results being checkpointed and reused, and service rejection being propagated
|
||||||
|
without creating a completion record. Nonempty valid-JSON output validation is
|
||||||
|
unchanged. Both new success regressions failed before the fix.
|
||||||
|
|
||||||
|
Location: `internal/manifest/transcribe_state.go:95`.
|
||||||
|
|
||||||
|
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
|
||||||
|
a regular `.flac` file and delegated transcription acceptance to WhisperX;
|
||||||
|
neither the user request nor the roadmap introduced minimum audio size or a new
|
||||||
|
audio-content validation policy. Zero-byte content still has a valid checksum
|
||||||
|
and can be represented by the checkpoint mechanism. The current check runs
|
||||||
|
after a successful response and output materialization, causing the entire
|
||||||
|
stage to fail instead of retaining the completed work.
|
||||||
|
|
||||||
|
Reproduction: prepare one speaker, replace its prepared audio with an empty
|
||||||
|
regular file, and use the existing successful fake WhisperX client. Stage
|
||||||
|
execution fails at checkpoint validation with `audio_size must be positive`.
|
||||||
|
This demonstrates the extra Narratio restriction; it does not assert that a
|
||||||
|
particular production WhisperX deployment accepts empty audio.
|
||||||
|
|
||||||
|
Required change:
|
||||||
|
|
||||||
|
- Permit zero audio size and reject negative sizes. Keep byte hashing and
|
||||||
|
size equality checks. Let the downstream service decide whether the input
|
||||||
|
can be transcribed, as before.
|
||||||
|
- Do not remove the nonempty valid-JSON output requirement, which has a separate
|
||||||
|
existing justification. Do not add a new early empty-audio rejection instead.
|
||||||
|
- Update any new tests/documentation that promote strictly positive input size
|
||||||
|
to a contract.
|
||||||
|
|
||||||
|
Verification: cover a zero-size record round-trip and successful adapter result
|
||||||
|
being checkpointed/reused for that input; retain negative-size rejection and
|
||||||
|
ordinary downstream-failure tests.
|
||||||
|
|
||||||
|
## Verification After Remediation
|
||||||
|
|
||||||
|
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
|
||||||
|
changed-package race tests, `go test ./...`, `go test -race ./...`,
|
||||||
|
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
|
||||||
|
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||||
|
`git diff --check` all passed. Existing replay and cancellation regression
|
||||||
|
tests remain intact; the HTTP adapter required no changes.
|
||||||
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
# Resumable Speaker Transcription and Replayable Uploads
|
||||||
|
|
||||||
|
Status: Implemented.
|
||||||
|
|
||||||
|
## Goal and User Intent
|
||||||
|
|
||||||
|
Preserve completed speaker transcriptions when another speaker fails, so an
|
||||||
|
ordinary run or resume does not repeat expensive successful work. Separately,
|
||||||
|
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
|
||||||
|
recoverable connection or stream failure.
|
||||||
|
|
||||||
|
The motivating incident exhausted four transcription attempts for one speaker
|
||||||
|
after three other speakers had completed. The final error reported an HTTP/2
|
||||||
|
peer `PROTOCOL_ERROR` and an inability to replay the request body without
|
||||||
|
`Request.GetBody`.
|
||||||
|
|
||||||
|
These are related reliability improvements, not a promise to eliminate all
|
||||||
|
service failures. This document owns the target state and scope; the
|
||||||
|
[implementation plan](implementation.md) owns ordered changes and concrete
|
||||||
|
storage and replay decisions.
|
||||||
|
|
||||||
|
## Pre-implementation Limitations
|
||||||
|
|
||||||
|
The [transcribe stage](../internal/stage-transcribe.md) writes successful
|
||||||
|
responses into the invocation's run directory. It installs canonical transcripts
|
||||||
|
and returns its per-file metadata only after every speaker succeeds. A failure
|
||||||
|
therefore leaves no durable per-speaker progress that ordinary resume can use,
|
||||||
|
even though some completed JSON files may remain on disk.
|
||||||
|
|
||||||
|
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
|
||||||
|
the application level, reopening audio for each attempt. Its multipart request
|
||||||
|
body is streamed through a pipe and has no replay callback. Transport-level
|
||||||
|
recovery within an attempt cannot reconstruct a consumed body. Providing that
|
||||||
|
capability addresses the replay limitation, not the underlying peer protocol
|
||||||
|
error; persistent failures can still require service or proxy investigation.
|
||||||
|
|
||||||
|
## Targeted End State
|
||||||
|
|
||||||
|
### Durable Per-Speaker Completion
|
||||||
|
|
||||||
|
Each successfully transcribed speaker file is validated, saved, and recorded
|
||||||
|
durably before that work is treated as reusable. The session manifest remains
|
||||||
|
the progress authority. Follow the existing analyze-stage checkpoint pattern
|
||||||
|
without turning speakers into pipeline stages or introducing a general task
|
||||||
|
engine.
|
||||||
|
|
||||||
|
Ordinary execution reconciles the current prepared audio set with recorded
|
||||||
|
speaker results. It reuses valid completed work and schedules failed,
|
||||||
|
unattempted, missing, or obsolete results. A result is not reusable merely
|
||||||
|
because a matching filename exists. Evidence binds it to the speaker/audio
|
||||||
|
identity, audio content, Narratio-visible recognition settings, and the saved
|
||||||
|
transcript's integrity.
|
||||||
|
|
||||||
|
The relevant recognition settings remain the WhisperX service URL and language.
|
||||||
|
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
|
||||||
|
remain operational settings rather than reasons to discard completed results.
|
||||||
|
Changes to private service models or configuration remain an operator reason
|
||||||
|
to force transcription; Narratio does not inspect the service's private state.
|
||||||
|
|
||||||
|
Changes to individual audio inputs require replacement of affected results,
|
||||||
|
not automatic retranscription of unrelated unchanged speakers. New speakers
|
||||||
|
need transcription; removed speakers do not remain in the current aggregate
|
||||||
|
output set. Records must describe the inputs actually used, not grant reuse to
|
||||||
|
old bytes under a changed identity.
|
||||||
|
|
||||||
|
### Aggregate Completion, Failure, and Force
|
||||||
|
|
||||||
|
The overall stage succeeds only when every currently required speaker has a
|
||||||
|
validated, durably recorded result. Incomplete per-speaker progress does not
|
||||||
|
make the aggregate stage successful or authorize downstream consumption of an
|
||||||
|
incomplete collection. Existing dependent-stage invalidation remains in effect.
|
||||||
|
|
||||||
|
Retain bounded concurrency and the current fail-fast behavior: a speaker
|
||||||
|
failure cancels outstanding work. Results already checkpointed remain reusable
|
||||||
|
after failure, cancellation, or process interruption. A later ordinary run or
|
||||||
|
resume retries the remaining work; it does not need a new CLI command.
|
||||||
|
|
||||||
|
Checkpoint writes must be serialized so concurrently finishing speakers cannot
|
||||||
|
overwrite each other's progress. A failed persistence operation must not
|
||||||
|
advertise an unrecorded result as complete. Previously committed progress remains
|
||||||
|
the recovery authority, and checkpoint errors must be surfaced rather than
|
||||||
|
reported as successful transcription.
|
||||||
|
|
||||||
|
`--force` on transcribe regenerates all currently required speakers instead of
|
||||||
|
reusing their prior completions. Successful replacements are checkpointed too,
|
||||||
|
so an ordinary retry after a failed forced invocation can retain the newly
|
||||||
|
completed work. Old results for speakers not successfully regenerated must not
|
||||||
|
satisfy that unfinished forced replacement on the next ordinary retry.
|
||||||
|
|
||||||
|
This deliberately relaxes the current all-speakers-before-canonical-installation
|
||||||
|
implementation: individual durable results may exist while the stage remains
|
||||||
|
failed or incomplete. The all-speakers-before-stage-success boundary remains.
|
||||||
|
|
||||||
|
### Replayable WhisperX Requests
|
||||||
|
|
||||||
|
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
|
||||||
|
same request. Replay reopens the audio and reproduces the multipart boundary,
|
||||||
|
filename, fields, and framing used by the original request. Independent bodies
|
||||||
|
must not share a consumed reader or mutable read position.
|
||||||
|
|
||||||
|
Preserve bounded-memory streaming rather than buffering entire audio files in
|
||||||
|
memory. Correctly close every original and replay body and its audio resources;
|
||||||
|
preserve source-read errors and cancellation, including partial uploads and
|
||||||
|
early server responses. Successful response handling must not accidentally wait
|
||||||
|
on, or report errors solely from, an abandoned body instead of the replay that
|
||||||
|
produced the response.
|
||||||
|
|
||||||
|
Keep the existing application-level retry count, delay, classification, and
|
||||||
|
per-attempt timeout semantics. Transport replays occur within an application
|
||||||
|
attempt and its context; they are not additional Narratio-configured attempts.
|
||||||
|
Do not add another custom retry loop, silently increase retries, or disable
|
||||||
|
HTTP/2 globally as a substitute for replay support.
|
||||||
|
|
||||||
|
The adapter still validates and installs only successful JSON responses.
|
||||||
|
Request replay does not provide exactly-once server execution: a lost response
|
||||||
|
can cause repeated transcription work, as can existing application retries.
|
||||||
|
No new service-side idempotency protocol is required by this feature.
|
||||||
|
|
||||||
|
## Required Change Areas
|
||||||
|
|
||||||
|
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
|
||||||
|
evidence, with safe checkpoint updates and preservation through runner failure
|
||||||
|
handling. Review manifest validation, copying, serialization, and existing
|
||||||
|
publish/restore handling wherever the new records pass through those paths.
|
||||||
|
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
|
||||||
|
checkpoint validated completions, rebuild aggregate outputs from the current
|
||||||
|
set, and account for partial progress in ordinary and forced execution.
|
||||||
|
Review successful-stage reuse as well as failed-stage retries so stale audio
|
||||||
|
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
|
||||||
|
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
|
||||||
|
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
|
||||||
|
directory surviving cleanup. Do not introduce a second authoritative ledger
|
||||||
|
or infer completion by scanning abandoned runs.
|
||||||
|
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
|
||||||
|
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
|
||||||
|
an upstream service change.
|
||||||
|
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
|
||||||
|
checkpoint failure; changed/missing inputs or outputs; force followed by an
|
||||||
|
ordinary retry; concurrent successful completions; and aggregate downstream
|
||||||
|
gating. Extend existing fixtures rather than duplicating analyze's complete
|
||||||
|
test suite. Exercise actual body reconstruction after partial consumption,
|
||||||
|
multipart equivalence, cancellation/resource cleanup, and existing bounded
|
||||||
|
streaming and retry behavior—not just the presence of a callback. Default
|
||||||
|
tests remain offline and use local/fake HTTP boundaries, never paid services.
|
||||||
|
- **Documentation:** Update the canonical
|
||||||
|
[stage guide](../internal/stage-transcribe.md),
|
||||||
|
[manifest guide](../internal/manifest.md),
|
||||||
|
[WhisperX contract](../integrations/whisperx.md),
|
||||||
|
[operations guide](../operations.md), and
|
||||||
|
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
|
||||||
|
partial completion, ordinary retry versus force, and the distinction between
|
||||||
|
request replay limitations and persistent peer protocol failures.
|
||||||
|
|
||||||
|
## Compatibility and Scope Boundaries
|
||||||
|
|
||||||
|
Existing speaker identities, raw transcript formats, and the fixed pipeline
|
||||||
|
order remain unchanged. Existing manifests remain readable, but must not be
|
||||||
|
treated as containing per-speaker evidence they never recorded. Recovery relies
|
||||||
|
on verified recorded evidence, not guessed associations between old transcripts
|
||||||
|
and current audio. No general legacy-migration framework is part of the feature.
|
||||||
|
|
||||||
|
Automatic adoption of unrecorded outputs from previously failed runs is outside
|
||||||
|
this feature. Such files may be recoverable after verifying their inputs, but
|
||||||
|
their presence alone does not prove completion. No recovery/import command,
|
||||||
|
speaker-selection CLI, historical invocation ledger, new service configuration
|
||||||
|
discovery, or broader orchestration refactor is included.
|
||||||
|
|
||||||
|
The feature does not diagnose the remote service/proxy's underlying HTTP/2
|
||||||
|
problem, guarantee recovery from persistent protocol errors, or change the
|
||||||
|
policy for continuing unrelated work after failure.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
|
||||||
|
completed, ordinary resume retains A and B, retries C, and completes D. No
|
||||||
|
downstream stage receives an incomplete successful transcription collection.
|
||||||
|
Force replaces all four, while an ordinary retry can reuse checkpoints from
|
||||||
|
that forced replacement attempt.
|
||||||
|
|
||||||
|
The HTTP transport can reconstruct the upload after consuming the original
|
||||||
|
body, without whole-file memory buffering, lost source errors, or stranded
|
||||||
|
upload resources. A persistent peer error still fails normally after the
|
||||||
|
configured application attempts, without discarding checkpointed speakers.
|
||||||
@@ -56,7 +56,8 @@ Symptom:
|
|||||||
|
|
||||||
Likely cause:
|
Likely cause:
|
||||||
|
|
||||||
- stale field name, typo, invalid enum, or invalid duration/path format.
|
- stale field name, typo, invalid enum, invalid duration/path format, or an
|
||||||
|
unsupported local manifest schema/state marker.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -68,9 +69,39 @@ Safe fix:
|
|||||||
|
|
||||||
- align config with [Configuration](./config.md) and the
|
- align config with [Configuration](./config.md) and the
|
||||||
[maintained examples](../examples/README.md).
|
[maintained examples](../examples/README.md).
|
||||||
|
- for a manifest schema error, move the unsupported local state aside and
|
||||||
|
regenerate it; restore a verified current committed snapshot or republish
|
||||||
|
healthy regenerated state when appropriate. Do not add version or stage
|
||||||
|
markers by hand.
|
||||||
|
- if an older manifest is rejected for its retired cleanup-obligation field,
|
||||||
|
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
|
||||||
|
remote commit, move aside or regenerate the incompatible manifest, and use
|
||||||
|
the cleanup stage. Do not treat manual JSON editing as the normal migration
|
||||||
|
path.
|
||||||
|
|
||||||
Relevant reference: [Configuration](./config.md).
|
Relevant reference: [Configuration](./config.md).
|
||||||
|
|
||||||
|
## Party schema version rejected
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- configuration loading reports that `party.schema_version` is required or
|
||||||
|
unsupported.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- the campaign `party.yml` is an unversioned legacy document, or names a
|
||||||
|
schema other than `narratio.party.v1`.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- replace the file with one exact, single-document
|
||||||
|
[`narratio.party.v1` party](./integrations/party.md); do not add a version
|
||||||
|
marker to an otherwise incompatible legacy structure. The versioned party is
|
||||||
|
the only configured roster authority.
|
||||||
|
|
||||||
|
Relevant reference: [Canonical Party Input](./integrations/party.md).
|
||||||
|
|
||||||
## Unexpected imported or profile value
|
## Unexpected imported or profile value
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -171,8 +202,9 @@ Safe fix:
|
|||||||
|
|
||||||
- widen the bounded range to include the first reported stage, or recover that
|
- widen the bounded range to include the first reported stage, or recover that
|
||||||
stage explicitly with `run-stage` before retrying. The failed check does not
|
stage explicitly with `run-stage` before retrying. The failed check does not
|
||||||
create a run record or modify the manifest. Narratio does not resume-validate
|
create a run-local directory or modify the manifest. Narratio does not
|
||||||
excluded prefix stages, and stages after `--through` are not prerequisites.
|
resume-validate excluded prefix stages, and stages after `--through` are not
|
||||||
|
prerequisites.
|
||||||
|
|
||||||
If prerequisite statuses are terminal but a selected stage reports a missing,
|
If prerequisite statuses are terminal but a selected stage reports a missing,
|
||||||
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
||||||
@@ -181,6 +213,134 @@ input validation.
|
|||||||
|
|
||||||
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
||||||
|
|
||||||
|
## Transcription retries speakers that previously finished
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
|
||||||
|
finish earlier; or
|
||||||
|
- an older successful session retranscribes all speakers once after upgrade.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the earlier response or canonical file was not followed by a successful
|
||||||
|
speaker manifest checkpoint;
|
||||||
|
- the prepared audio bytes/path, WhisperX URL, or language changed;
|
||||||
|
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
|
||||||
|
no longer matches its recorded size/checksum;
|
||||||
|
- the previous invocation used `--force`, which durably cleared pre-force
|
||||||
|
speaker authority; or
|
||||||
|
- the manifest has only legacy aggregate transcribe state and cannot bind old
|
||||||
|
outputs to their source audio.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- inspect the reported speaker and filesystem error, repair the prepared input
|
||||||
|
or canonical-output path when needed, then retry transcribe normally;
|
||||||
|
- allow a legacy aggregate-only record to regenerate once; and
|
||||||
|
- do not edit the manifest or copy an old JSON file into place as a substitute
|
||||||
|
for a successful checkpoint.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Valid earlier speaker checkpoints remain reusable even while aggregate
|
||||||
|
transcribe status is failed or interrupted. Merge remains gated until
|
||||||
|
transcribe reaches aggregate success with the complete current set.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
|
||||||
|
|
||||||
|
## WhisperX model changed but transcription was reused
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- WhisperX model weights, prompts, or private server configuration changed at
|
||||||
|
the same configured URL, but Narratio still considers speaker transcripts
|
||||||
|
current.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- Narratio can fingerprint its configured URL and language, not private state
|
||||||
|
loaded behind the service boundary.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- explicitly force transcribe to establish a durable reset, then use ordinary
|
||||||
|
retry if the replacement attempt is interrupted.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Transport replay and application retries do not provide an exactly-once remote
|
||||||
|
execution guarantee. Investigate server-side failures or duplicate work using
|
||||||
|
WhisperX diagnostics; do not infer completion from a server log without the
|
||||||
|
local speaker checkpoint.
|
||||||
|
|
||||||
|
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
|
||||||
|
[Transcribe internals](./internal/stage-transcribe.md).
|
||||||
|
|
||||||
|
## Transcription manifest checkpoint failed
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- transcribe reports that a speaker completed but the session manifest could
|
||||||
|
not be saved.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the workspace is unwritable or out of space;
|
||||||
|
- the manifest path or parent directory is unsafe or the wrong type; or
|
||||||
|
- a transient filesystem error interrupted atomic replacement.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct the reported filesystem or permission problem and retry transcribe
|
||||||
|
normally. Narratio reconciles from the last readable collection and reuses
|
||||||
|
only successfully checkpointed speakers.
|
||||||
|
|
||||||
|
A canonical raw transcript left by the failed checkpoint is incidental and
|
||||||
|
does not become reusable authority. Preserve the manifest and diagnostics if
|
||||||
|
the failure repeats; do not advertise the file by editing manifest evidence.
|
||||||
|
|
||||||
|
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
|
||||||
|
and [Transcribe internals](./internal/stage-transcribe.md).
|
||||||
|
|
||||||
|
## Notarius credential reference unavailable
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
|
||||||
|
name for `pipeline.notarius.llm_api_key`, including during inspection or a
|
||||||
|
run that would not execute extraction.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the named environment variable is absent or empty;
|
||||||
|
- the selected secrets directory lacks a file with that exact name;
|
||||||
|
- a process environment variable exists but is empty, so bootstrap correctly
|
||||||
|
leaves it in place; or
|
||||||
|
- the Narratio reference name does not match `api_key_env` in the selected
|
||||||
|
Notarius/PromptKit profile.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- set the exact nonempty environment variable or provide an equivalently named
|
||||||
|
file through the conventional or explicit secrets directory;
|
||||||
|
- remove or set the field to `null` when that deployment needs no credential;
|
||||||
|
and
|
||||||
|
- use `config show`, `config sources`, and `config diff` to inspect the public
|
||||||
|
reference name and ownership without exposing its value.
|
||||||
|
|
||||||
|
Narratio resolves selected credential references during ordinary pipeline
|
||||||
|
loading, before stage selection or resume. Credential changes alone do not
|
||||||
|
invalidate a completed extraction, but the current reference must still resolve
|
||||||
|
for the command to proceed.
|
||||||
|
|
||||||
|
Relevant references: [Secrets](./config.md#secrets-handling) and
|
||||||
|
[Notarius credential](./config.md#notarius-credential).
|
||||||
|
|
||||||
## Notarius executable missing
|
## Notarius executable missing
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -257,6 +417,59 @@ Do not point Notarius directly at the original source path or edit the manifest
|
|||||||
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
||||||
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius reference snapshot cache conflict
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction reports that a path beneath
|
||||||
|
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
|
||||||
|
non-regular or unsafe entry.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- stop commands for the affected session and inspect the exact reported path;
|
||||||
|
- investigate unexpected links or special files as possible tampering; and
|
||||||
|
- remove only the reported conflicting leaf or digest directory after
|
||||||
|
confirming it belongs to the reconstructible reference cache.
|
||||||
|
|
||||||
|
A missing snapshot or corrupt regular snapshot is reconstructed automatically
|
||||||
|
from currently verified prepared input. Do not recursively delete the session
|
||||||
|
root, edit the manifest checksum, or remove cache entries while extraction is
|
||||||
|
active.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius resume executes work instead of reusing it
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Narratio executes `extract`, and Notarius makes provider calls despite
|
||||||
|
`pipeline.notarius.resume: true`.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- checkpoint recording is disabled in the Notarius configuration;
|
||||||
|
- Notarius is using a nonpersistent or different checkpoint root;
|
||||||
|
- transcript/reference bytes, reference paths, pipeline/profile content,
|
||||||
|
local-backend endpoint, selected lanes, or another checkpoint identity input
|
||||||
|
changed; or
|
||||||
|
- Narratio extraction was forced, which deliberately omits `--resume`.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- validate the deployed Notarius configuration and configure explicit
|
||||||
|
persistent chunk-plan and checkpoint roots;
|
||||||
|
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
|
||||||
|
and
|
||||||
|
- retry without force only when reuse is intended.
|
||||||
|
|
||||||
|
Remember that Narratio skipping a completed `extract` stage is separate from
|
||||||
|
Notarius reusing work inside an extraction that does execute. Narratio does not
|
||||||
|
read the Notarius configuration to infer checkpoint enablement.
|
||||||
|
|
||||||
|
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
|
||||||
|
and [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
## Notarius reference selector or generated-handoff collision
|
## Notarius reference selector or generated-handoff collision
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -282,29 +495,6 @@ Narratio validates selector structure and prepared evidence, while Notarius
|
|||||||
owns slot declarations, media compatibility, and generated-handoff conflicts.
|
owns slot declarations, media compatibility, and generated-handoff conflicts.
|
||||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
## Atomic Notarius promotion unsupported
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
|
||||||
before a durable bundle or temporary promotion tree is created.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Narratio is running on an operating system other than Linux, macOS, or
|
|
||||||
Windows, where the required atomic no-replace directory primitive has not
|
|
||||||
been implemented and verified.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
|
||||||
with a manual copy or move; the session manifest must never observe a partial
|
|
||||||
or overwritten bundle.
|
|
||||||
|
|
||||||
This is an extraction-specific platform boundary, not a support statement for
|
|
||||||
unrelated Narratio workflows. See
|
|
||||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
|
||||||
|
|
||||||
## Notarius receipt or index incompatible
|
## Notarius receipt or index incompatible
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -352,18 +542,23 @@ Symptom:
|
|||||||
|
|
||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
- the pipeline ID or configured output contracts changed;
|
||||||
configured output contracts changed;
|
|
||||||
- a configured prepared reference selector, source, path, checksum, or byte
|
- a configured prepared reference selector, source, path, checksum, or byte
|
||||||
size changed;
|
size changed;
|
||||||
|
- the direct trimmed transcript identity or bytes changed; or
|
||||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||||
checksum no longer validates.
|
checksum no longer validates.
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- allow the automatic rerun after verifying the current configuration. Treat
|
- allow the automatic rerun after verifying the current configuration. Treat
|
||||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
||||||
investigate it rather than replacing files manually.
|
and investigate it rather than replacing files manually.
|
||||||
|
|
||||||
|
Executable, config path, timeout, working directory, and Notarius credential
|
||||||
|
changes are operational and do not by themselves invalidate reusable extraction
|
||||||
|
evidence. A newly selected credential must nevertheless resolve during command
|
||||||
|
configuration loading.
|
||||||
|
|
||||||
## Notarius transitive configuration changed
|
## Notarius transitive configuration changed
|
||||||
|
|
||||||
@@ -395,8 +590,7 @@ Symptom:
|
|||||||
|
|
||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- the per-artifact record is stale, missing, failed, unselected, malformed, or
|
- the per-artifact record is stale, missing, failed, unselected, or malformed;
|
||||||
from the legacy aggregate-only manifest contract;
|
|
||||||
- a configured prompt/profile, dependency, input identity, output path, or
|
- a configured prompt/profile, dependency, input identity, output path, or
|
||||||
effective variable changed; or
|
effective variable changed; or
|
||||||
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
|
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
|
||||||
@@ -425,36 +619,89 @@ narratio analyze 2026-04-04 --artifacts session_recap
|
|||||||
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
|
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
|
||||||
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
|
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
|
||||||
|
|
||||||
## Legacy aggregate analysis requires regeneration
|
## Analysis manifest checkpoint failed
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
|
|
||||||
- a manifest from an older Narratio version reports aggregate analyze success
|
- analyze reports that it could not save the session manifest after completing
|
||||||
and the old files are present, but configured artifact sources remain
|
an artifact.
|
||||||
unavailable.
|
|
||||||
|
|
||||||
Likely cause:
|
Likely causes:
|
||||||
|
|
||||||
- the manifest has no supported per-artifact analyze state. Aggregate output
|
- the workspace is unwritable or out of space;
|
||||||
lists do not establish current configured-artifact authority.
|
- the manifest path or its parent directory has unsafe permissions or type; or
|
||||||
|
- a transient local filesystem error interrupted atomic manifest replacement.
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- regenerate the required artifacts. A partial selection makes only its
|
- correct the reported filesystem or permission problem;
|
||||||
targets and prerequisites eligible for current state; unselected legacy
|
- inspect `session artifacts` to identify the last durable current records; and
|
||||||
files intentionally remain unavailable. Run full analysis later when every
|
- rerun analyze normally. Narratio reuses earlier successful checkpoints and
|
||||||
enabled configured artifact must become current.
|
retries the failed or non-current artifact and its required dependents.
|
||||||
|
|
||||||
```bash
|
Do not treat a newly created canonical artifact file as current when its
|
||||||
narratio analyze 2026-04-04 --artifacts session_recap
|
checkpoint failed, and do not edit the manifest to advertise it. If the error
|
||||||
narratio analyze 2026-04-04
|
recurs, preserve the manifest and run-local diagnostics before repairing the
|
||||||
```
|
workspace.
|
||||||
|
|
||||||
After current records exist, inspect them and publish explicitly. Do not delete
|
|
||||||
the legacy files merely to influence selection; availability is manifest-owned.
|
|
||||||
|
|
||||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||||
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state).
|
and [Manifest Internals](./internal/manifest.md#persistence-semantics).
|
||||||
|
|
||||||
|
## Built-in transcript or bounds source is unavailable
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- analyze, extract, render, or publish says a built-in source is unavailable
|
||||||
|
even though a file exists at its usual session path.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- the producing stage has not succeeded, its output lacks the exact registered
|
||||||
|
source, run ID, or checksum, its evidence is duplicated or stale, or the
|
||||||
|
durable bytes no longer match the manifest checksum.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- do not copy or rename the file to make it discoverable. Rerun the producing
|
||||||
|
stage to regenerate both the bytes and current manifest evidence, then retry
|
||||||
|
the consumer.
|
||||||
|
|
||||||
|
## An older manifest schema is rejected
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Narratio rejects a local session manifest because `schema_version` is missing
|
||||||
|
or unsupported; or
|
||||||
|
- restore rejects a committed remote session manifest for the same reason.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the manifest predates the supported canonical state contract. Narratio
|
||||||
|
rejects it at the schema boundary rather than interpreting aggregate output
|
||||||
|
lists or incidental files as current authority; or
|
||||||
|
- an older strict-decoding Narratio binary is reading a current manifest that
|
||||||
|
contains additive transcribe speaker state it does not recognize.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- for unsupported local state, move the old manifest aside and regenerate, or
|
||||||
|
restore a verified current committed snapshot when one is authoritative;
|
||||||
|
- for unsupported committed remote state, regenerate healthy local state as
|
||||||
|
needed and republish the session; and
|
||||||
|
- for a downgrade rejection, return to the current binary or restore a verified
|
||||||
|
pre-upgrade backup. Do not delete only the speaker fields by hand.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run 2026-04-04
|
||||||
|
narratio publish 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
After current records exist, inspect them and publish explicitly. Do not copy
|
||||||
|
old files into canonical paths merely to influence selection; availability is
|
||||||
|
manifest-owned.
|
||||||
|
|
||||||
|
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||||
|
and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
|
||||||
|
|
||||||
## Scriptorium private input changed without a rerun
|
## Scriptorium private input changed without a rerun
|
||||||
|
|
||||||
@@ -491,7 +738,9 @@ Symptom:
|
|||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- missing `session.previous_session_id`;
|
- missing `session.previous_session_id`;
|
||||||
- previous artifact not restored/published for source session.
|
- the selected previous-session commit does not declare the required canonical
|
||||||
|
source as a `published_output`; or
|
||||||
|
- the declared published object is unavailable or fails identity validation.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -549,6 +798,8 @@ Symptom:
|
|||||||
Likely cause:
|
Likely cause:
|
||||||
|
|
||||||
- local durable files differ from remote restore sources.
|
- local durable files differ from remote restore sources.
|
||||||
|
- a restore target or one of its ancestors is a symlink or another non-regular
|
||||||
|
filesystem entry.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -559,7 +810,11 @@ narratio session restore 2026-04-04 --dry-run
|
|||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- review conflicts;
|
- review conflicts;
|
||||||
- rerun with `--force` only when remote state should overwrite local.
|
- rerun with `--force` only when remote state should overwrite an eligible
|
||||||
|
regular local file; repair a symlink or other non-regular conflict first.
|
||||||
|
|
||||||
|
If the filesystem changed after the plan was printed, rerun restore after
|
||||||
|
repairing the destination; installation performs the same safety check again.
|
||||||
|
|
||||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||||
|
|
||||||
@@ -652,7 +907,8 @@ Symptom:
|
|||||||
|
|
||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- invalid `pipeline.secrets.env_dir`;
|
- missing, unreadable, or invalid explicit `--secrets-dir`;
|
||||||
|
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
|
||||||
- missing credential environment variables;
|
- missing credential environment variables;
|
||||||
- invalid S3 endpoint/bucket settings.
|
- invalid S3 endpoint/bucket settings.
|
||||||
|
|
||||||
@@ -660,12 +916,12 @@ Diagnostics:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
ls -la /path/to/secrets_dir
|
ls -la /path/to/secrets_dir
|
||||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- correct secret-file path and permissions;
|
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
|
||||||
- provide required env vars;
|
- provide required env vars;
|
||||||
- keep secret values out of YAML.
|
- keep secret values out of YAML.
|
||||||
|
|
||||||
|
|||||||
@@ -11,18 +11,26 @@ in the [configuration reference](../docs/config.md).
|
|||||||
WhisperX URL.
|
WhisperX URL.
|
||||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
|
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
|
||||||
`conf.d` imports, a production default, and selectable production/testing
|
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
|
||||||
overlays. It also demonstrates canonical-party artifact families and a
|
one family per `artifact-families/<key>.yml`, and the fixed
|
||||||
testing-only disabled artifact. Validate it with:
|
`profiles/testing.yml` overlay. Production values live in the base. The
|
||||||
|
bundle demonstrates an input-only artifact dependency, two character families
|
||||||
|
sharing the campaign party, and a testing-only disabled artifact. Inspect it
|
||||||
|
offline with:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
`config show` and `config sources` accept the same selectors and remain
|
These inspection commands load configuration and resolve explicit credential
|
||||||
read-only.
|
references, but do not load a session or create runtime state. This bundle is
|
||||||
|
intentionally local-storage and credential-reference free so the walkthrough
|
||||||
|
needs no tools or cloud services.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
@@ -52,11 +60,20 @@ with the sample campaign and a compatible local- or S3-audio session.
|
|||||||
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
||||||
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||||
follows the Notarius v0.6 contract. Narratio derives the players projection
|
follows the Notarius v0.7 contract. Narratio derives the players projection
|
||||||
from this party source; the campaign deliberately has no `players_file`.
|
from this party source.
|
||||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||||
reserves the expected filename and directory shape. Replace it with a real
|
reserves the expected filename and directory shape. Replace it with a real
|
||||||
FLAC file before running transcription.
|
FLAC file before running transcription.
|
||||||
|
|
||||||
The examples contain environment-variable names but no credential values. They
|
The examples contain environment-variable names but no credential values. Set
|
||||||
use fictional campaign content and reserved example domains.
|
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
|
||||||
|
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
|
||||||
|
examples that reference them. They use fictional campaign content and reserved
|
||||||
|
example domains.
|
||||||
|
|
||||||
|
For a deployed bundle, keep credential files out of source control in a
|
||||||
|
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
|
||||||
|
environment-variable name, or select another directory with `--secrets-dir`.
|
||||||
|
Static S3 credentials are optional: omit both S3 credential references to use
|
||||||
|
the AWS SDK discovery chain.
|
||||||
|
|||||||
@@ -7,9 +7,8 @@ workspace:
|
|||||||
# Optional: remove run-scoped workdir after successful publish commit.
|
# Optional: remove run-scoped workdir after successful publish commit.
|
||||||
cleanup_after_publish: false
|
cleanup_after_publish: false
|
||||||
|
|
||||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
# Credential files, when used, belong in a sibling secrets/ directory or in the
|
||||||
# secrets:
|
# directory selected with --secrets-dir; they are not pipeline configuration.
|
||||||
# env_dir: ./secrets
|
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||||
@@ -24,8 +23,8 @@ storage:
|
|||||||
endpoint: ""
|
endpoint: ""
|
||||||
force_path_style: false
|
force_path_style: false
|
||||||
# Optional; defaults shown explicitly.
|
# Optional; defaults shown explicitly.
|
||||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
secret_access_key: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||||
@@ -94,7 +93,7 @@ audita:
|
|||||||
# Optional overrides; defaults shown explicitly where applicable.
|
# Optional overrides; defaults shown explicitly where applicable.
|
||||||
binary: audita
|
binary: audita
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
llm_api_key: AUDITA_LLM_API_KEY
|
||||||
modules: [glossary, homophones, spoken_word, grammar]
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
base_url: ""
|
base_url: ""
|
||||||
model: ""
|
model: ""
|
||||||
@@ -129,13 +128,26 @@ trim:
|
|||||||
report: false
|
report: false
|
||||||
|
|
||||||
notarius:
|
notarius:
|
||||||
# Optional structured extraction between trim and render.
|
# Optional structured extraction after render and before analyze.
|
||||||
enabled: true
|
enabled: true
|
||||||
binary: notarius
|
binary: notarius
|
||||||
|
# Optional environment-variable reference. Match this name to api_key_env in
|
||||||
|
# the selected Notarius/PromptKit profile; never put the credential here.
|
||||||
|
llm_api_key: NOTARIUS_LLM_API_KEY
|
||||||
|
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
|
||||||
|
# is independent of Narratio's selected profile and may use a different name.
|
||||||
|
# See docs/config.md and docs/integrations/notarius.md for the contracts.
|
||||||
|
# config_profile: downstream-economy
|
||||||
config_path: /usr/local/etc/notarius/config.yml
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
pipeline_id: dnd-session
|
pipeline_id: dnd-session
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
working_directory: /usr/local/etc/notarius
|
working_directory: /usr/local/etc/notarius
|
||||||
|
# Reuse compatible accepted Notarius checkpoints during executed extraction.
|
||||||
|
# The Notarius config must enable recording and provide persistent cache roots.
|
||||||
|
resume: true
|
||||||
|
# auto uses or creates a source-addressed chunk plan. Forced Narratio
|
||||||
|
# extraction overrides this with refresh and disables checkpoint loading.
|
||||||
|
chunk_cache: auto
|
||||||
# External campaign references use prepared Narratio source IDs. Omit an
|
# External campaign references use prepared Narratio source IDs. Omit an
|
||||||
# optional binding when the selected Notarius pipeline does not need it.
|
# optional binding when the selected Notarius pipeline does not need it.
|
||||||
references:
|
references:
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ storage:
|
|||||||
bucket: my-dnd-archive
|
bucket: my-dnd-archive
|
||||||
root_prefix: dnd
|
root_prefix: dnd
|
||||||
region: us-east-1
|
region: us-east-1
|
||||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
secret_access_key: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
root: /usr/local/share/narratio/campaigns
|
root: /usr/local/share/narratio/campaigns
|
||||||
@@ -57,7 +57,7 @@ seriatim:
|
|||||||
audita:
|
audita:
|
||||||
binary: audita
|
binary: audita
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
llm_api_key: AUDITA_LLM_API_KEY
|
||||||
modules: [glossary, homophones, spoken_word, grammar]
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
output_schema: audita-v1
|
output_schema: audita-v1
|
||||||
work_dir_retention: auto
|
work_dir_retention: auto
|
||||||
|
|||||||
@@ -1,23 +1,11 @@
|
|||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
|
||||||
config_path: ./scriptorium/config.yml
|
|
||||||
artifact_families:
|
artifact_families:
|
||||||
character_meta:
|
|
||||||
enabled: true
|
|
||||||
for_each: party.characters
|
|
||||||
prompt_id: dnd.character_meta
|
|
||||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
|
||||||
member_vars:
|
|
||||||
character_id: character_id
|
|
||||||
character_name: character.name
|
|
||||||
player_name: player.name
|
|
||||||
class_summary: character.class_summary
|
|
||||||
character_items:
|
character_items:
|
||||||
enabled: true
|
enabled: true
|
||||||
for_each: party.characters
|
for_each: party.characters
|
||||||
prompt_id: dnd.character_items
|
prompt_id: dnd.character_items
|
||||||
|
profile_id: production-placeholder
|
||||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||||
member_dependencies: [character_meta]
|
|
||||||
inputs:
|
inputs:
|
||||||
character_meta:
|
character_meta:
|
||||||
source: narratio.member_artifact.character_meta
|
source: narratio.member_artifact.character_meta
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
enabled: true
|
||||||
|
for_each: party.characters
|
||||||
|
prompt_id: dnd.character_meta
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||||
|
member_vars:
|
||||||
|
character_id: character_id
|
||||||
|
character_name: character.name
|
||||||
|
player_name: player.name
|
||||||
|
class_summary: character.class_summary
|
||||||
11
examples/production-testing/artifacts/player_handout.yml
Normal file
11
examples/production-testing/artifacts/player_handout.yml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
player_handout:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.player_handout
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path: artifacts/player-handout.md
|
||||||
|
inputs:
|
||||||
|
recap:
|
||||||
|
source: narratio.artifact.session_recap
|
||||||
|
required: true
|
||||||
11
examples/production-testing/artifacts/session_recap.yml
Normal file
11
examples/production-testing/artifacts/session_recap.yml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path: artifacts/session-recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
@@ -10,6 +10,8 @@ audita:
|
|||||||
binary: audita
|
binary: audita
|
||||||
modules: [glossary, grammar]
|
modules: [glossary, grammar]
|
||||||
output_schema: audita-v1
|
output_schema: audita-v1
|
||||||
|
model: narratio-production-model-placeholder
|
||||||
|
validation_model: narratio-production-validator-placeholder
|
||||||
|
|
||||||
normalize:
|
normalize:
|
||||||
output_path: transcripts/final.json
|
output_path: transcripts/final.json
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
# Copyable production/testing pipeline entry point. Every fragment is named
|
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
|
||||||
# explicitly; Narratio never scans conf.d automatically.
|
# these explicit path and glob imports; Narratio never scans directories itself.
|
||||||
composition:
|
configkit:
|
||||||
imports:
|
imports:
|
||||||
- conf.d/platform.yml
|
- path: conf.d/platform.yml
|
||||||
- conf.d/storage.yml
|
- path: conf.d/storage.yml
|
||||||
- conf.d/transcript.yml
|
- path: conf.d/transcript.yml
|
||||||
- conf.d/artifacts.yml
|
- path: conf.d/publish.yml
|
||||||
- conf.d/publish.yml
|
- glob: artifacts/*.yml
|
||||||
default_profile: production
|
- glob: artifact-families/*.yml
|
||||||
profiles:
|
|
||||||
production:
|
scriptorium:
|
||||||
overlay: profiles/production.yml
|
binary: scriptorium
|
||||||
testing:
|
config_path: ./scriptorium/config.yml
|
||||||
overlay: profiles/testing.yml
|
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
audita:
|
|
||||||
model: narratio-production-model-placeholder
|
|
||||||
validation_model: narratio-production-validator-placeholder
|
|
||||||
|
|
||||||
scriptorium:
|
|
||||||
artifact_families:
|
|
||||||
character_meta:
|
|
||||||
profile_id: production-placeholder
|
|
||||||
character_items:
|
|
||||||
profile_id: production-placeholder
|
|
||||||
@@ -4,6 +4,10 @@ audita:
|
|||||||
|
|
||||||
scriptorium:
|
scriptorium:
|
||||||
artifacts:
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
profile_id: testing-placeholder
|
||||||
|
player_handout:
|
||||||
|
profile_id: testing-placeholder
|
||||||
testing_notes:
|
testing_notes:
|
||||||
enabled: false
|
enabled: false
|
||||||
output_path: artifacts/testing-notes.md
|
output_path: artifacts/testing-notes.md
|
||||||
|
|||||||
6
go.mod
6
go.mod
@@ -1,12 +1,15 @@
|
|||||||
module gitea.maximumdirect.net/eric/narratio
|
module gitea.maximumdirect.net/eric/narratio
|
||||||
|
|
||||||
go 1.25.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
github.com/aws/smithy-go v1.25.1
|
github.com/aws/smithy-go v1.25.1
|
||||||
|
github.com/google/go-cmp v0.7.0
|
||||||
|
golang.org/x/sync v0.22.0
|
||||||
golang.org/x/sys v0.47.0
|
golang.org/x/sys v0.47.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
@@ -26,4 +29,5 @@ require (
|
|||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -1,3 +1,5 @@
|
|||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1 h1:etfTI/dxO3NnS18La0V6IeUh/Iv+3PjVfEyaN8hTuOM=
|
||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||||
@@ -34,6 +36,12 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
|||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -84,17 +83,17 @@ func materializePlaceholders(req PolishRequest) error {
|
|||||||
"merged_transcript_path": req.MergedTranscriptPath,
|
"merged_transcript_path": req.MergedTranscriptPath,
|
||||||
"output_path": req.OutputProcessedPath,
|
"output_path": req.OutputProcessedPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -124,7 +123,7 @@ func writeJSONIfRequested(path string, payload any) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
19
internal/adapters/audita/generated_config.go
Normal file
19
internal/adapters/audita/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package audita
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeGeneratedYAML(path string, payload any) error {
|
||||||
|
data, err := yaml.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
|
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -24,7 +23,8 @@ const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
|||||||
type SubprocessRunnerConfig struct {
|
type SubprocessRunnerConfig struct {
|
||||||
Binary string
|
Binary string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
LLMAPIKeyEnv string
|
LLMAPIKey string
|
||||||
|
LLMAPIKeyReference string
|
||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
@@ -43,7 +43,8 @@ type SubprocessRunnerConfig struct {
|
|||||||
type SubprocessRunner struct {
|
type SubprocessRunner struct {
|
||||||
binary string
|
binary string
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
llmAPIKeyEnv string
|
llmAPIKey string
|
||||||
|
llmAPIKeyReference string
|
||||||
modules []string
|
modules []string
|
||||||
baseURL string
|
baseURL string
|
||||||
model string
|
model string
|
||||||
@@ -62,7 +63,8 @@ type SubprocessRunner struct {
|
|||||||
func NewSubprocessRunnerFromConfigValues(
|
func NewSubprocessRunnerFromConfigValues(
|
||||||
binary string,
|
binary string,
|
||||||
timeout string,
|
timeout string,
|
||||||
llmAPIKeyEnv string,
|
llmAPIKey string,
|
||||||
|
llmAPIKeyReference string,
|
||||||
modules []string,
|
modules []string,
|
||||||
baseURL string,
|
baseURL string,
|
||||||
model string,
|
model string,
|
||||||
@@ -86,7 +88,8 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
return NewSubprocessRunner(SubprocessRunnerConfig{
|
return NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: binary,
|
Binary: binary,
|
||||||
Timeout: parsedTimeout,
|
Timeout: parsedTimeout,
|
||||||
LLMAPIKeyEnv: llmAPIKeyEnv,
|
LLMAPIKey: llmAPIKey,
|
||||||
|
LLMAPIKeyReference: llmAPIKeyReference,
|
||||||
Modules: modules,
|
Modules: modules,
|
||||||
BaseURL: baseURL,
|
BaseURL: baseURL,
|
||||||
Model: model,
|
Model: model,
|
||||||
@@ -110,6 +113,9 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||||
}
|
}
|
||||||
|
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
|
||||||
|
return nil, fmt.Errorf("audita credential value and reference must be provided together")
|
||||||
|
}
|
||||||
for i, module := range cfg.Modules {
|
for i, module := range cfg.Modules {
|
||||||
if strings.TrimSpace(module) == "" {
|
if strings.TrimSpace(module) == "" {
|
||||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||||
@@ -152,7 +158,8 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
return &SubprocessRunner{
|
return &SubprocessRunner{
|
||||||
binary: strings.TrimSpace(cfg.Binary),
|
binary: strings.TrimSpace(cfg.Binary),
|
||||||
timeout: cfg.Timeout,
|
timeout: cfg.Timeout,
|
||||||
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
|
llmAPIKey: cfg.LLMAPIKey,
|
||||||
|
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
|
||||||
modules: modules,
|
modules: modules,
|
||||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||||
model: strings.TrimSpace(cfg.Model),
|
model: strings.TrimSpace(cfg.Model),
|
||||||
@@ -195,16 +202,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
args := r.buildArgs(req, reqModules)
|
args := r.buildArgs(req, reqModules)
|
||||||
|
|
||||||
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
|
credentialPresent := r.llmAPIKeyReference != ""
|
||||||
credentialPresent := false
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
if credentialEnvVar != "" {
|
if credentialPresent {
|
||||||
credential, ok := os.LookupEnv(credentialEnvVar)
|
env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
|
||||||
if !ok || strings.TrimSpace(credential) == "" {
|
|
||||||
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
|
|
||||||
}
|
|
||||||
env["AUDITA_LLM_API_KEY"] = credential
|
|
||||||
credentialPresent = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.GeneratedConfigPath != "" {
|
if req.GeneratedConfigPath != "" {
|
||||||
@@ -269,7 +270,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -299,7 +300,7 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
|||||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -375,10 +376,10 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
|||||||
"output_path": req.OutputProcessedPath,
|
"output_path": req.OutputProcessedPath,
|
||||||
"report_path": req.ReportPath,
|
"report_path": req.ReportPath,
|
||||||
"work_dir": req.WorkDir,
|
"work_dir": req.WorkDir,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProcessedOutput(path string) error {
|
func validateProcessedOutput(path string) error {
|
||||||
@@ -425,7 +426,7 @@ func validateJSONFile(path string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||||
data, err := fileops.ReadRegularFile(path, limit)
|
data, err := fileops.ReadFileBounded(path, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
@@ -31,7 +27,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: wrapper,
|
Binary: wrapper,
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary", "homophones", "glossary"},
|
Modules: []string{"glossary", "homophones", "glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
@@ -48,6 +45,7 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||||
}
|
}
|
||||||
|
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := PolishRequest{
|
req := PolishRequest{
|
||||||
@@ -132,47 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
}
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
|
||||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
|
||||||
Modules: []string{"glossary"},
|
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||||
|
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||||
_, err := runner.Run(context.Background(), req)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error, got nil")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
|
|
||||||
t.Fatalf("error = %q, want missing credential guidance", err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
|
|
||||||
@@ -190,9 +172,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||||
@@ -200,13 +179,13 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
|||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -220,21 +199,18 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -249,19 +225,16 @@ func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -279,22 +252,20 @@ func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -310,25 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(err.Error(), "super-secret") {
|
||||||
|
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -341,22 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -369,22 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -397,22 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -425,22 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -453,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected binary validation error")
|
t.Fatal("expected binary validation error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected timeout parse error")
|
t.Fatal("expected timeout parse error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: "audita",
|
Binary: "audita",
|
||||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||||
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
|
LLMAPIKey: "synthetic-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||||
BaseURL: "://",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "://",
|
||||||
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected base url validation error")
|
t.Fatal("expected base url validation error")
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error)
|
|||||||
return RunResult{}, err
|
return RunResult{}, err
|
||||||
}
|
}
|
||||||
copyRequest := req
|
copyRequest := req
|
||||||
|
if req.ConfigProfile != nil {
|
||||||
|
profile := *req.ConfigProfile
|
||||||
|
copyRequest.ConfigProfile = &profile
|
||||||
|
}
|
||||||
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
|
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
|
||||||
f.Requests = append(f.Requests, copyRequest)
|
f.Requests = append(f.Requests, copyRequest)
|
||||||
if f.Err != nil {
|
if f.Err != nil {
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ package notarius
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
||||||
@@ -22,7 +24,10 @@ type ReferenceBinding struct {
|
|||||||
|
|
||||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
Binary string
|
Binary string
|
||||||
|
// LLMAPIKey is an optional already-resolved credential reference.
|
||||||
|
LLMAPIKey *configkit.Secret
|
||||||
|
ConfigProfile *string
|
||||||
ConfigPath string
|
ConfigPath string
|
||||||
PipelineID string
|
PipelineID string
|
||||||
InputPath string
|
InputPath string
|
||||||
@@ -31,6 +36,8 @@ type RunRequest struct {
|
|||||||
ReceiptPath string
|
ReceiptPath string
|
||||||
LogPath string
|
LogPath string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
|
Resume bool
|
||||||
|
ChunkCache string
|
||||||
References []ReferenceBinding
|
References []ReferenceBinding
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -52,6 +52,20 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return RunResult{}, err
|
return RunResult{}, err
|
||||||
}
|
}
|
||||||
|
var envOverrides map[string]string
|
||||||
|
var sensitiveEnvNames []string
|
||||||
|
if req.LLMAPIKey != nil {
|
||||||
|
if !req.LLMAPIKey.IsResolved() {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||||
|
}
|
||||||
|
name := req.LLMAPIKey.EnvName()
|
||||||
|
value := req.LLMAPIKey.Value()
|
||||||
|
if value == "" {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||||
|
}
|
||||||
|
envOverrides = map[string]string{name: value}
|
||||||
|
sensitiveEnvNames = []string{name}
|
||||||
|
}
|
||||||
|
|
||||||
args := []string{
|
args := []string{
|
||||||
"run", req.PipelineID,
|
"run", req.PipelineID,
|
||||||
@@ -59,18 +73,29 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
|||||||
"--input", req.InputPath,
|
"--input", req.InputPath,
|
||||||
"--output-dir", req.OutputRoot,
|
"--output-dir", req.OutputRoot,
|
||||||
}
|
}
|
||||||
|
if req.Resume {
|
||||||
|
args = append(args, "--resume")
|
||||||
|
}
|
||||||
|
if req.ChunkCache != "" {
|
||||||
|
args = append(args, "--chunk_cache", req.ChunkCache)
|
||||||
|
}
|
||||||
for _, reference := range references {
|
for _, reference := range references {
|
||||||
args = append(args, "--reference", reference.Selector+"="+reference.Path)
|
args = append(args, "--reference", reference.Selector+"="+reference.Path)
|
||||||
}
|
}
|
||||||
|
if req.ConfigProfile != nil {
|
||||||
|
args = append(args, "--config-profile", *req.ConfigProfile)
|
||||||
|
}
|
||||||
args = append(args, "--json")
|
args = append(args, "--json")
|
||||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||||
Executable: req.Binary,
|
Executable: req.Binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
WorkingDir: req.WorkingDirectory,
|
WorkingDir: req.WorkingDirectory,
|
||||||
Timeout: req.Timeout,
|
Timeout: req.Timeout,
|
||||||
DiagnosticOwner: "notarius",
|
DiagnosticOwner: "notarius",
|
||||||
StdoutLogPath: req.ReceiptPath,
|
StdoutLogPath: req.ReceiptPath,
|
||||||
StderrLogPath: req.LogPath,
|
StderrLogPath: req.LogPath,
|
||||||
|
EnvOverrides: envOverrides,
|
||||||
|
SensitiveEnvNames: sensitiveEnvNames,
|
||||||
})
|
})
|
||||||
baseResult := RunResult{
|
baseResult := RunResult{
|
||||||
ReceiptPath: req.ReceiptPath,
|
ReceiptPath: req.ReceiptPath,
|
||||||
@@ -143,6 +168,9 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
|||||||
if req.Timeout <= 0 {
|
if req.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("notarius timeout must be positive")
|
return nil, fmt.Errorf("notarius timeout must be positive")
|
||||||
}
|
}
|
||||||
|
if !validChunkCacheMode(req.ChunkCache) {
|
||||||
|
return nil, fmt.Errorf("notarius chunk cache mode must be empty, auto, bypass, or refresh")
|
||||||
|
}
|
||||||
for label, path := range map[string]string{
|
for label, path := range map[string]string{
|
||||||
"config": req.ConfigPath,
|
"config": req.ConfigPath,
|
||||||
"input": req.InputPath,
|
"input": req.InputPath,
|
||||||
@@ -201,6 +229,15 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
|||||||
return references, nil
|
return references, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validChunkCacheMode(mode string) bool {
|
||||||
|
switch mode {
|
||||||
|
case "", "auto", "bypass", "refresh":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type receiptDocument struct {
|
type receiptDocument struct {
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
@@ -248,7 +285,7 @@ func validateValidationSummaries(documents []validationSummaryDocument) ([]Valid
|
|||||||
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
||||||
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
||||||
}
|
}
|
||||||
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) {
|
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount < 0 || !validTerminalAction(document.TerminalAction) {
|
||||||
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
||||||
}
|
}
|
||||||
summaries = append(summaries, ValidationSummary{
|
summaries = append(summaries, ValidationSummary{
|
||||||
@@ -671,7 +708,7 @@ func sumWarningOccurrences(values []WarningSummary) (int, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||||
data, err := fileops.ReadRegularFile(path, limit)
|
data, err := fileops.ReadFileBounded(path, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -76,6 +77,174 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
resume bool
|
||||||
|
chunkCache string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{name: "resume", resume: true, want: []string{"--resume"}},
|
||||||
|
{name: "auto", chunkCache: "auto", want: []string{"--chunk_cache", "auto"}},
|
||||||
|
{name: "bypass", chunkCache: "bypass", want: []string{"--chunk_cache", "bypass"}},
|
||||||
|
{name: "refresh", chunkCache: "refresh", want: []string{"--chunk_cache", "refresh"}},
|
||||||
|
{name: "resume and mode", resume: true, chunkCache: "auto", want: []string{"--resume", "--chunk_cache", "auto"}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.Resume = test.resume
|
||||||
|
req.ChunkCache = test.chunkCache
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot,
|
||||||
|
}
|
||||||
|
want = append(want, test.want...)
|
||||||
|
want = append(want, "--json")
|
||||||
|
if !reflect.DeepEqual(captured.Args, want) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerMapsConfigProfileToArgumentPair(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
supplied bool
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{name: "omitted"},
|
||||||
|
{name: "empty", supplied: true},
|
||||||
|
{name: "default", supplied: true, value: "default"},
|
||||||
|
{name: "named", supplied: true, value: "inexpensive"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
if test.supplied {
|
||||||
|
req.ConfigProfile = &test.value
|
||||||
|
}
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
flagIndex := -1
|
||||||
|
for index, arg := range captured.Args {
|
||||||
|
if arg == "--config-profile" {
|
||||||
|
flagIndex = index
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !test.supplied {
|
||||||
|
if flagIndex >= 0 {
|
||||||
|
t.Fatalf("subprocess args unexpectedly contain config profile: %#v", captured.Args)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if flagIndex < 0 || flagIndex+2 >= len(captured.Args) || captured.Args[flagIndex+1] != test.value || captured.Args[flagIndex+2] != "--json" {
|
||||||
|
t.Fatalf("config profile argument boundary = %#v, want flag, exact value, then --json", captured.Args)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
|
||||||
|
for _, mode := range []string{"AUTO", "invalid", " auto "} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.ChunkCache = mode
|
||||||
|
started := false
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
started = true
|
||||||
|
return sharedsubprocess.RunResult{}, nil
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "chunk cache mode") {
|
||||||
|
t.Fatalf("Run() error = %v, want chunk cache validation failure", err)
|
||||||
|
}
|
||||||
|
if started {
|
||||||
|
t.Fatal("subprocess started with invalid chunk cache mode")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerForwardsLoadedCredentialWithoutRereadingEnvironment(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_LLM_KEY"
|
||||||
|
const loadedValue = "loaded-notarius-credential"
|
||||||
|
credential := loadTestCredential(t, envName, loadedValue)
|
||||||
|
t.Setenv(envName, "changed-parent-credential")
|
||||||
|
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = credential
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(captured.EnvOverrides) != 1 || captured.EnvOverrides[envName] != loadedValue {
|
||||||
|
t.Fatal("subprocess did not receive exactly the loaded credential override")
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{envName}) {
|
||||||
|
t.Fatalf("sensitive environment names = %#v, want configured reference name", captured.SensitiveEnvNames)
|
||||||
|
}
|
||||||
|
wantArgs := []string{
|
||||||
|
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot, "--json",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRejectsInvalidProgrammaticCredentialBeforeLaunch(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
reference configkit.Secret
|
||||||
|
}{
|
||||||
|
{name: "unresolved", reference: configkit.SecretEnv("NARRATIO_NOTARIUS_UNRESOLVED_KEY")},
|
||||||
|
{name: "empty", reference: configkit.SecretEnv("")},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = &test.reference
|
||||||
|
started := false
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
started = true
|
||||||
|
return sharedsubprocess.RunResult{}, nil
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "resolved and nonempty") {
|
||||||
|
t.Fatalf("Run() error = %v, want credential validation failure", err)
|
||||||
|
}
|
||||||
|
if started {
|
||||||
|
t.Fatal("subprocess started with an invalid credential")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
|
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
referenceRoot := t.TempDir()
|
referenceRoot := t.TempDir()
|
||||||
@@ -172,16 +341,23 @@ func TestSubprocessRunnerRejectsInvalidReferencesBeforeLaunch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_KEY"
|
||||||
|
const canary = "notarius-captured-stream-canary"
|
||||||
|
const unrelated = "NARRATIO_NOTARIUS_UNRELATED_PARENT_KEY"
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||||
writeValidBundleAndReceipt(t, req, false)
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
receiptFixture := req.ReceiptPath + ".fixture"
|
receiptFixture := req.ReceiptPath + ".fixture"
|
||||||
data, err := os.ReadFile(req.ReceiptPath)
|
data, err := os.ReadFile(req.ReceiptPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
var receipt map[string]any
|
||||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
if err := json.Unmarshal(data, &receipt); err != nil {
|
||||||
|
t.Fatalf("decode receipt fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
receipt["credential_echo"] = canary
|
||||||
|
writeJSONFile(t, receiptFixture, receipt)
|
||||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||||
t.Fatalf("Remove(receipt) error = %v", err)
|
t.Fatalf("Remove(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -193,27 +369,55 @@ func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T)
|
|||||||
script := writeShellScript(t, `#!/bin/sh
|
script := writeShellScript(t, `#!/bin/sh
|
||||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||||
printf 'diagnostic stream\n' >&2
|
if [ "$`+envName+`" != '`+canary+`' ]; then
|
||||||
|
printf 'selected credential mismatch\n' >&2
|
||||||
|
exit 8
|
||||||
|
fi
|
||||||
|
if [ -n "$`+unrelated+`" ]; then
|
||||||
|
printf 'unrelated credential inherited\n' >&2
|
||||||
|
exit 9
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$`+envName+`" >&2
|
||||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||||
`)
|
`)
|
||||||
req.Binary = script
|
req.Binary = script
|
||||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||||
|
t.Setenv(unrelated, "unrelated-parent-credential")
|
||||||
|
|
||||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
||||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
assertRedactedCredentialBytes(t, receiptBytes, canary)
|
||||||
t.Fatal("receipt contains stderr output")
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRedactsCredentialOnProcessFailure(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_KEY"
|
||||||
|
const canary = "notarius-failure-stream-canary"
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||||
|
req.Binary = writeShellScript(t, `#!/bin/sh
|
||||||
|
printf '%s\n' "$`+envName+`"
|
||||||
|
printf '%s\n' "$`+envName+`" >&2
|
||||||
|
exit 7
|
||||||
|
`)
|
||||||
|
_, err := NewSubprocessRunner().Run(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want process failure")
|
||||||
}
|
}
|
||||||
|
if strings.Contains(err.Error(), canary) || !strings.Contains(err.Error(), "<redacted>") {
|
||||||
|
t.Fatal("process failure did not return redacted diagnostics")
|
||||||
|
}
|
||||||
|
assertRedactedCredentialFile(t, req.ReceiptPath, canary)
|
||||||
|
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||||
@@ -231,6 +435,10 @@ func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T
|
|||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
|
if test.name == "nonzero" {
|
||||||
|
profile := "missing-profile"
|
||||||
|
req.ConfigProfile = &profile
|
||||||
|
}
|
||||||
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||||
req.Timeout = test.timeout
|
req.Timeout = test.timeout
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -527,7 +735,8 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||||
req := RunRequest{PipelineID: "pipeline", References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
|
profile := "testing"
|
||||||
|
req := RunRequest{PipelineID: "pipeline", ConfigProfile: &profile, References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
|
||||||
want := RunResult{BundleRoot: "/bundle"}
|
want := RunResult{BundleRoot: "/bundle"}
|
||||||
fake := &FakeRunner{Result: want}
|
fake := &FakeRunner{Result: want}
|
||||||
got, err := fake.Run(context.Background(), req)
|
got, err := fake.Run(context.Background(), req)
|
||||||
@@ -538,6 +747,10 @@ func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
|||||||
if fake.Requests[0].References[0].Path != "/references/party.json" {
|
if fake.Requests[0].References[0].Path != "/references/party.json" {
|
||||||
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
|
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
|
||||||
}
|
}
|
||||||
|
profile = "changed"
|
||||||
|
if *fake.Requests[0].ConfigProfile != "testing" {
|
||||||
|
t.Fatalf("fake retained aliased config profile: %q", *fake.Requests[0].ConfigProfile)
|
||||||
|
}
|
||||||
|
|
||||||
wantErr := errors.New("configured failure")
|
wantErr := errors.New("configured failure")
|
||||||
fake.Err = wantErr
|
fake.Err = wantErr
|
||||||
@@ -579,6 +792,56 @@ func validRunRequest(t *testing.T) RunRequest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadTestCredential(t *testing.T, envName, value string) *configkit.Secret {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(envName, value)
|
||||||
|
type testConfig struct {
|
||||||
|
Credential *configkit.Secret `yaml:"credential"`
|
||||||
|
}
|
||||||
|
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+envName+"\n")), configkit.Options[testConfig]{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load test credential: %v", err)
|
||||||
|
}
|
||||||
|
return result.Config.Credential
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertRedactedCredentialFile(t *testing.T, path, canary string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
assertRedactedCredentialBytes(t, data, canary)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertRedactedCredentialBytes(t *testing.T, data []byte, canary string) {
|
||||||
|
t.Helper()
|
||||||
|
text := string(data)
|
||||||
|
if strings.Contains(text, canary) || !strings.Contains(text, "<redacted>") {
|
||||||
|
t.Fatal("captured stream did not contain the required redaction marker")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateValidationSummariesAcceptsReusedProducerWithoutAttempts(t *testing.T) {
|
||||||
|
zero := 0
|
||||||
|
summaries, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||||
|
Stage: "chunk", Status: "complete", ProducerAttemptCount: &zero, TerminalAction: "accepted",
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateValidationSummaries() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(summaries) != 1 || summaries[0].ProducerAttemptCount != 0 {
|
||||||
|
t.Fatalf("summaries = %#v, want one reused producer with zero attempts", summaries)
|
||||||
|
}
|
||||||
|
|
||||||
|
negative := -1
|
||||||
|
if _, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||||
|
Stage: "chunk", Status: "complete", ProducerAttemptCount: &negative, TerminalAction: "accepted",
|
||||||
|
}}); err == nil {
|
||||||
|
t.Fatal("validateValidationSummaries() negative producer attempts error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -143,7 +142,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
|
|||||||
|
|
||||||
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||||
if req.OutputPath != "" {
|
if req.OutputPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -155,17 +154,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
|||||||
"prompt_id": req.PromptID,
|
"prompt_id": req.PromptID,
|
||||||
"output_path": req.OutputPath,
|
"output_path": req.OutputPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -174,7 +173,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
|||||||
|
|
||||||
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||||
if req.OutputPath != "" {
|
if req.OutputPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,17 +185,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
|||||||
"prompt_id": req.PromptID,
|
"prompt_id": req.PromptID,
|
||||||
"output_path": req.OutputPath,
|
"output_path": req.OutputPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
19
internal/adapters/scriptorium/generated_config.go
Normal file
19
internal/adapters/scriptorium/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package scriptorium
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeGeneratedYAML(path string, payload any) error {
|
||||||
|
data, err := yaml.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
|
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -342,11 +342,11 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
|
|||||||
"render_format": payload.RenderFormat,
|
"render_format": payload.RenderFormat,
|
||||||
"render_prompt_logged": payload.RenderPromptStore,
|
"render_prompt_logged": payload.RenderPromptStore,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(path, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateNonEmptyOutput(path string) error {
|
func validateNonEmptyOutput(path string) error {
|
||||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
||||||
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -261,7 +260,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
|
|||||||
|
|
||||||
func materializePlaceholders(req MergeRequest) error {
|
func materializePlaceholders(req MergeRequest) error {
|
||||||
if req.OutputMergedTranscriptPath != "" {
|
if req.OutputMergedTranscriptPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -272,22 +271,22 @@ func materializePlaceholders(req MergeRequest) error {
|
|||||||
"input_transcript_paths": req.InputTranscriptPaths,
|
"input_transcript_paths": req.InputTranscriptPaths,
|
||||||
"output_path": req.OutputMergedTranscriptPath,
|
"output_path": req.OutputMergedTranscriptPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -296,7 +295,7 @@ func materializePlaceholders(req MergeRequest) error {
|
|||||||
|
|
||||||
func materializeTrimPlaceholders(req TrimRequest) error {
|
func materializeTrimPlaceholders(req TrimRequest) error {
|
||||||
if req.OutputTrimmedPath != "" {
|
if req.OutputTrimmedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -309,17 +308,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
|||||||
"output_path": req.OutputTrimmedPath,
|
"output_path": req.OutputTrimmedPath,
|
||||||
"keep_selector": req.KeepSelector,
|
"keep_selector": req.KeepSelector,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -328,7 +327,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
|||||||
|
|
||||||
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||||
if req.OutputNormalizedPath != "" {
|
if req.OutputNormalizedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -344,22 +343,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
|||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
payload["report_path"] = req.ReportPath
|
payload["report_path"] = req.ReportPath
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -368,7 +367,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
|||||||
|
|
||||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||||
if req.OutputRenderedPath != "" {
|
if req.OutputRenderedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -385,17 +384,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
|
|||||||
"include_segment_ids": req.IncludeSegmentIDs,
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
"include_metadata": req.IncludeMetadata,
|
"include_metadata": req.IncludeMetadata,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
19
internal/adapters/seriatim/generated_config.go
Normal file
19
internal/adapters/seriatim/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package seriatim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeGeneratedYAML(path string, payload any) error {
|
||||||
|
data, err := yaml.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
|
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -553,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
|
|||||||
payload["coalesce_gap"] = *r.coalesceGap
|
payload["coalesce_gap"] = *r.coalesceGap
|
||||||
}
|
}
|
||||||
|
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildTrimArgs(req TrimRequest) []string {
|
func buildTrimArgs(req TrimRequest) []string {
|
||||||
@@ -605,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
|
|||||||
"output_path": req.OutputTrimmedPath,
|
"output_path": req.OutputTrimmedPath,
|
||||||
"keep_selector": req.KeepSelector,
|
"keep_selector": req.KeepSelector,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
||||||
@@ -620,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
|||||||
"output_schema": outputSchema,
|
"output_schema": outputSchema,
|
||||||
"report_path": req.ReportPath,
|
"report_path": req.ReportPath,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||||
@@ -638,7 +638,7 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
|
|||||||
"include_segment_ids": req.IncludeSegmentIDs,
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
"include_metadata": req.IncludeMetadata,
|
"include_metadata": req.IncludeMetadata,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateJSONFile(path string) error {
|
func validateJSONFile(path string) error {
|
||||||
@@ -692,7 +692,7 @@ func validateNonEmptyTextFile(path string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func readSeriatimResult(path, category string) ([]byte, error) {
|
func readSeriatimResult(path, category string) ([]byte, error) {
|
||||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
||||||
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
||||||
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
||||||
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|||||||
@@ -19,14 +19,17 @@ import (
|
|||||||
type FakeBackend struct {
|
type FakeBackend struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
|
|
||||||
Objects map[string]FakeObject
|
Objects map[string]FakeObject
|
||||||
Uploads []FakeUploadCall
|
Uploads []FakeUploadCall
|
||||||
Downloads []FakeDownloadCall
|
Downloads []FakeDownloadCall
|
||||||
Reads []FakeReadCall
|
Reads []FakeReadCall
|
||||||
|
Deletes []string
|
||||||
|
ExistsKeys []string
|
||||||
|
|
||||||
ListErr error
|
ListErr error
|
||||||
DownloadErr error
|
DownloadErr error
|
||||||
UploadErr error
|
UploadErr error
|
||||||
|
DeleteErr error
|
||||||
ExistsErr error
|
ExistsErr error
|
||||||
UploadHook func(FakeUploadCall) error
|
UploadHook func(FakeUploadCall) error
|
||||||
DownloadHook func(FakeDownloadCall) error
|
DownloadHook func(FakeDownloadCall) error
|
||||||
@@ -315,6 +318,25 @@ func fakeObjectETag(data []byte) string {
|
|||||||
return hex.EncodeToString(sum[:])
|
return hex.EncodeToString(sum[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete removes one object. Removing an absent object succeeds.
|
||||||
|
func (f *FakeBackend) Delete(ctx context.Context, key string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if f.DeleteErr != nil {
|
||||||
|
return f.DeleteErr
|
||||||
|
}
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return fmt.Errorf("delete object: key is required")
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.Deletes = append(f.Deletes, normalizedKey)
|
||||||
|
delete(f.Objects, normalizedKey)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Exists checks object presence.
|
// Exists checks object presence.
|
||||||
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -323,8 +345,12 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
|||||||
if f.ExistsErr != nil {
|
if f.ExistsErr != nil {
|
||||||
return false, f.ExistsErr
|
return false, f.ExistsErr
|
||||||
}
|
}
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
f.mu.Lock()
|
||||||
|
f.ExistsKeys = append(f.ExistsKeys, normalizedKey)
|
||||||
|
f.mu.Unlock()
|
||||||
f.mu.RLock()
|
f.mu.RLock()
|
||||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
_, ok := f.Objects[normalizedKey]
|
||||||
f.mu.RUnlock()
|
f.mu.RUnlock()
|
||||||
return ok, nil
|
return ok, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -71,23 +71,55 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendDeleteIsIdempotent(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
fake.SeedObject(FakeObject{Key: "locks/source.lock", Data: []byte("lock")})
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
if err := fake.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||||
|
t.Fatalf("Delete() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects["locks/source.lock"]; ok {
|
||||||
|
t.Fatal("Delete() left the object present")
|
||||||
|
}
|
||||||
|
if len(fake.Deletes) != 2 || fake.Deletes[0] != "locks/source.lock" || fake.Deletes[1] != "locks/source.lock" {
|
||||||
|
t.Fatalf("Delete() calls = %#v, want two normalized keys", fake.Deletes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendDeleteHonorsContextAndInjectedError(t *testing.T) {
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
if err := fake.Delete(canceled, "locks/source.lock"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Delete() canceled error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantErr := errors.New("delete fail")
|
||||||
|
fake.DeleteErr = wantErr
|
||||||
|
if err := fake.Delete(context.Background(), "locks/source.lock"); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Delete() injected error = %v, want %v", err, wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
||||||
fake := &FakeBackend{}
|
fake := &FakeBackend{}
|
||||||
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
|
fake.SeedObject(FakeObject{Key: "control.json", Data: []byte("old")})
|
||||||
old := fake.Objects["locks.yml"].ETag
|
old := fake.Objects["control.json"].ETag
|
||||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||||
t.Fatalf("UploadConditional() error = %v", err)
|
t.Fatalf("UploadConditional() error = %v", err)
|
||||||
}
|
}
|
||||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||||
}
|
}
|
||||||
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
|
if got := string(fake.Objects["control.json"].Data); got != "new" {
|
||||||
t.Fatalf("locks object = %q, want successful replacement preserved", got)
|
t.Fatalf("control object = %q, want successful replacement preserved", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), DeleteErr: errors.New("delete fail"), ExistsErr: errors.New("exists fail")}
|
||||||
|
|
||||||
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||||
t.Fatalf("List() error = %v, want list fail", err)
|
t.Fatalf("List() error = %v, want list fail", err)
|
||||||
@@ -100,6 +132,9 @@ func TestFakeBackendObjectErrors(t *testing.T) {
|
|||||||
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||||
t.Fatalf("Upload() error = %v, want upload fail", err)
|
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||||
}
|
}
|
||||||
|
if err := fake.Delete(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "delete fail") {
|
||||||
|
t.Fatalf("Delete() error = %v, want delete fail", err)
|
||||||
|
}
|
||||||
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||||
t.Fatalf("Exists() error = %v, want exists fail", err)
|
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ type ObjectStore interface {
|
|||||||
Download(ctx context.Context, key, localPath string) error
|
Download(ctx context.Context, key, localPath string) error
|
||||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||||
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
||||||
|
Delete(ctx context.Context, key string) error
|
||||||
Exists(ctx context.Context, key string) (bool, error)
|
Exists(ctx context.Context, key string) (bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ type s3API interface {
|
|||||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||||
|
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
|
||||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,8 +37,12 @@ type s3ClientOptions struct {
|
|||||||
Region string
|
Region string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
ForcePathStyle bool
|
ForcePathStyle bool
|
||||||
AccessKeyID string
|
Credentials *s3StaticCredentials
|
||||||
SecretKey string
|
}
|
||||||
|
|
||||||
|
type s3StaticCredentials struct {
|
||||||
|
AccessKeyID string
|
||||||
|
SecretAccessKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
@@ -45,11 +50,11 @@ var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error)
|
|||||||
if strings.TrimSpace(opts.Region) != "" {
|
if strings.TrimSpace(opts.Region) != "" {
|
||||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
if opts.Credentials != nil {
|
||||||
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||||
credentials.NewStaticCredentialsProvider(
|
credentials.NewStaticCredentialsProvider(
|
||||||
strings.TrimSpace(opts.AccessKeyID),
|
opts.Credentials.AccessKeyID,
|
||||||
strings.TrimSpace(opts.SecretKey),
|
opts.Credentials.SecretAccessKey,
|
||||||
"",
|
"",
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
@@ -74,14 +79,24 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
|||||||
if bucket == "" {
|
if bucket == "" {
|
||||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||||
}
|
}
|
||||||
|
if (cfg.AccessKeyID == nil) != (cfg.SecretAccessKey == nil) {
|
||||||
|
return nil, fmt.Errorf("storage.s3 access key ID and secret access key references must be configured together")
|
||||||
|
}
|
||||||
|
if cfg.AccessKeyID != nil && (!cfg.AccessKeyID.IsResolved() || !cfg.SecretAccessKey.IsResolved()) {
|
||||||
|
return nil, fmt.Errorf("storage.s3 configured credential references must be resolved")
|
||||||
|
}
|
||||||
|
|
||||||
client, err := newS3Client(ctx, s3ClientOptions{
|
clientOptions := s3ClientOptions{
|
||||||
Region: cfg.Region,
|
Region: cfg.Region,
|
||||||
Endpoint: cfg.Endpoint,
|
Endpoint: cfg.Endpoint,
|
||||||
ForcePathStyle: cfg.ForcePathStyle,
|
ForcePathStyle: cfg.ForcePathStyle,
|
||||||
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
}
|
||||||
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
if cfg.AccessKeyID != nil && cfg.SecretAccessKey != nil {
|
||||||
})
|
clientOptions.Credentials = &s3StaticCredentials{
|
||||||
|
AccessKeyID: cfg.AccessKeyID.Value(), SecretAccessKey: cfg.SecretAccessKey.Value(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client, err := newS3Client(ctx, clientOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||||
}
|
}
|
||||||
@@ -92,26 +107,6 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func s3CredentialFromEnv(envVarName string) string {
|
|
||||||
name := strings.TrimSpace(envVarName)
|
|
||||||
if name == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
value, ok := os.LookupEnv(name)
|
|
||||||
if !ok {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func orDefaultEnvName(name, fallback string) string {
|
|
||||||
trimmed := strings.TrimSpace(name)
|
|
||||||
if trimmed == "" {
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
return trimmed
|
|
||||||
}
|
|
||||||
|
|
||||||
// List returns objects under prefix.
|
// List returns objects under prefix.
|
||||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
normalizedPrefix := normalizeObjectKey(prefix)
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
@@ -300,6 +295,25 @@ func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key stri
|
|||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete removes one object. S3 deletion is idempotent when the key is absent.
|
||||||
|
func (b *S3Backend) Delete(ctx context.Context, key string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return fmt.Errorf("delete object: key is required")
|
||||||
|
}
|
||||||
|
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("delete object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Exists checks whether one object key exists.
|
// Exists checks whether one object key exists.
|
||||||
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
normalizedKey := normalizeObjectKey(key)
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
"github.com/aws/smithy-go"
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,16 +30,18 @@ type fakeS3API struct {
|
|||||||
getETag *string
|
getETag *string
|
||||||
getLastModified *time.Time
|
getLastModified *time.Time
|
||||||
|
|
||||||
putOut *s3.PutObjectOutput
|
putOut *s3.PutObjectOutput
|
||||||
putErr error
|
putErr error
|
||||||
|
deleteErr error
|
||||||
|
|
||||||
headErr error
|
headErr error
|
||||||
|
|
||||||
lastList *s3.ListObjectsV2Input
|
lastList *s3.ListObjectsV2Input
|
||||||
lastLists []*s3.ListObjectsV2Input
|
lastLists []*s3.ListObjectsV2Input
|
||||||
lastGet *s3.GetObjectInput
|
lastGet *s3.GetObjectInput
|
||||||
lastPut *s3.PutObjectInput
|
lastPut *s3.PutObjectInput
|
||||||
lastHead *s3.HeadObjectInput
|
lastDelete *s3.DeleteObjectInput
|
||||||
|
lastHead *s3.HeadObjectInput
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||||
@@ -116,6 +119,14 @@ func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ..
|
|||||||
return f.putOut, nil
|
return f.putOut, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) DeleteObject(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
|
||||||
|
f.lastDelete = params
|
||||||
|
if f.deleteErr != nil {
|
||||||
|
return nil, f.deleteErr
|
||||||
|
}
|
||||||
|
return &s3.DeleteObjectOutput{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||||
f.lastHead = params
|
f.lastHead = params
|
||||||
if f.headErr != nil {
|
if f.headErr != nil {
|
||||||
@@ -177,14 +188,14 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
|||||||
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||||
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
||||||
client := &fakeS3API{
|
client := &fakeS3API{
|
||||||
getBody: io.NopCloser(strings.NewReader("locks")),
|
getBody: io.NopCloser(strings.NewReader("state")),
|
||||||
getSize: int64Ptr(5),
|
getSize: int64Ptr(5),
|
||||||
getETag: strPtr(`"generation"`),
|
getETag: strPtr(`"generation"`),
|
||||||
getLastModified: &lastModified,
|
getLastModified: &lastModified,
|
||||||
}
|
}
|
||||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
|
info, body, err := backend.Read(context.Background(), `sessions\control.json`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Read() error = %v", err)
|
t.Fatalf("Read() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -193,10 +204,10 @@ func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
|||||||
if readErr != nil || closeErr != nil {
|
if readErr != nil || closeErr != nil {
|
||||||
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
||||||
}
|
}
|
||||||
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
if info.Key != "sessions/control.json" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||||
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
||||||
}
|
}
|
||||||
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
|
if string(data) != "state" || client.lastGet == nil || *client.lastGet.Key != "sessions/control.json" {
|
||||||
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,14 +249,14 @@ func TestS3BackendUploadAndExists(t *testing.T) {
|
|||||||
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
||||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||||
t.Fatalf("UploadConditional() error = %v", err)
|
t.Fatalf("UploadConditional() error = %v", err)
|
||||||
}
|
}
|
||||||
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
||||||
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
||||||
}
|
}
|
||||||
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
||||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||||
if !errors.Is(err, ErrConditionNotMet) {
|
if !errors.Is(err, ErrConditionNotMet) {
|
||||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||||
}
|
}
|
||||||
@@ -275,11 +286,44 @@ func TestS3BackendExistsNotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestS3BackendDeleteMapsRequestAndErrors(t *testing.T) {
|
||||||
|
client := &fakeS3API{}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
if err := backend.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||||
|
t.Fatalf("Delete() error = %v", err)
|
||||||
|
}
|
||||||
|
if client.lastDelete == nil || *client.lastDelete.Bucket != "bucket-1" || *client.lastDelete.Key != "locks/source.lock" {
|
||||||
|
t.Fatalf("DeleteObject input = %#v, want normalized bucket and key", client.lastDelete)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantErr := errors.New("permission denied")
|
||||||
|
client.deleteErr = wantErr
|
||||||
|
err := backend.Delete(context.Background(), "locks/source.lock")
|
||||||
|
if !errors.Is(err, wantErr) || !strings.Contains(err.Error(), "locks/source.lock") {
|
||||||
|
t.Fatalf("Delete() error = %v, want contextual backend error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
client := &fakeS3API{}
|
||||||
|
err := (&S3Backend{bucket: "bucket-1", client: client}).Delete(ctx, "locks/source.lock")
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Delete() error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
if client.lastDelete != nil {
|
||||||
|
t.Fatal("Delete() called S3 after cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||||
original := newS3Client
|
original := newS3Client
|
||||||
t.Cleanup(func() { newS3Client = original })
|
t.Cleanup(func() { newS3Client = original })
|
||||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
accessKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_ACCESS", "id-123")
|
||||||
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
secretKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_SECRET", "secret-abc")
|
||||||
|
t.Setenv("NARRATIO_S3_TEST_ACCESS", "changed-id")
|
||||||
|
t.Setenv("NARRATIO_S3_TEST_SECRET", "changed-secret")
|
||||||
|
|
||||||
var got s3ClientOptions
|
var got s3ClientOptions
|
||||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
@@ -288,10 +332,12 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
Bucket: "my-archive",
|
Bucket: "my-archive",
|
||||||
Region: "us-east-1",
|
Region: "us-east-1",
|
||||||
Endpoint: "http://localhost:9000",
|
Endpoint: "http://localhost:9000",
|
||||||
ForcePathStyle: true,
|
ForcePathStyle: true,
|
||||||
|
AccessKeyID: accessKey,
|
||||||
|
SecretAccessKey: secretKey,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
@@ -302,7 +348,7 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
|||||||
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||||
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||||
}
|
}
|
||||||
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
if got.Credentials == nil || got.Credentials.AccessKeyID != "id-123" || got.Credentials.SecretAccessKey != "secret-abc" {
|
||||||
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -314,7 +360,21 @@ func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
func TestNewS3BackendFromConfigRejectsIncompleteOrUnresolvedCredentials(t *testing.T) {
|
||||||
|
unresolved := configkit.SecretEnv("NARRATIO_UNRESOLVED_S3_KEY")
|
||||||
|
for name, cfg := range map[string]config.StorageS3Config{
|
||||||
|
"partial": {Bucket: "my-archive", AccessKeyID: &unresolved},
|
||||||
|
"unresolved": {Bucket: "my-archive", AccessKeyID: &unresolved, SecretAccessKey: &unresolved},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := NewS3BackendFromConfig(context.Background(), cfg); err == nil {
|
||||||
|
t.Fatal("NewS3BackendFromConfig() succeeded, want credential error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigUsesDefaultProviderWithoutCredentialReferences(t *testing.T) {
|
||||||
original := newS3Client
|
original := newS3Client
|
||||||
t.Cleanup(func() { newS3Client = original })
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
|
||||||
@@ -325,19 +385,30 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
Bucket: "my-archive",
|
Bucket: "my-archive",
|
||||||
Region: "us-east-1",
|
Region: "us-east-1",
|
||||||
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
|
||||||
SecretKeyEnv: "MISSING_SECRET_KEY",
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
}
|
}
|
||||||
if got.AccessKeyID != "" || got.SecretKey != "" {
|
if got.Credentials != nil {
|
||||||
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
t.Fatalf("client options credentials = %#v, want default provider chain", got.Credentials)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolvedTestSecret(t *testing.T, name, value string) *configkit.Secret {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(name, value)
|
||||||
|
type document struct {
|
||||||
|
Secret *configkit.Secret `yaml:"secret"`
|
||||||
|
}
|
||||||
|
result, err := configkit.Load(configkit.Bytes("test.yml", []byte("secret: "+name+"\n")), configkit.Options[document]{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve test secret: %v", err)
|
||||||
|
}
|
||||||
|
return result.Config.Secret
|
||||||
|
}
|
||||||
|
|
||||||
func strPtr(v string) *string { return &v }
|
func strPtr(v string) *string { return &v }
|
||||||
func int64Ptr(v int64) *int64 { return &v }
|
func int64Ptr(v int64) *int64 { return &v }
|
||||||
func boolPtr(v bool) *bool { return &v }
|
func boolPtr(v bool) *bool { return &v }
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ import (
|
|||||||
|
|
||||||
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||||
store := &FakeBackend{}
|
store := &FakeBackend{}
|
||||||
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
store.SeedObject(FakeObject{Key: "sessions/a/runs/run-123/metadata.txt", Data: []byte("run-123\n")})
|
||||||
|
|
||||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -38,7 +38,7 @@ func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
|||||||
t.Fatalf("Glob(before) error = %v", err)
|
t.Fatalf("Glob(before) error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", pattern)
|
||||||
if !errors.Is(err, sentinel) {
|
if !errors.Is(err, sentinel) {
|
||||||
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||||
}
|
}
|
||||||
@@ -58,7 +58,7 @@ func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
|||||||
sentinel := errors.New("object missing")
|
sentinel := errors.New("object missing")
|
||||||
store := &FakeBackend{DownloadErr: sentinel}
|
store := &FakeBackend{DownloadErr: sentinel}
|
||||||
|
|
||||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ type logWriters struct {
|
|||||||
Stdout io.Writer
|
Stdout io.Writer
|
||||||
Stderr io.Writer
|
Stderr io.Writer
|
||||||
|
|
||||||
limits chan *captureLimitError
|
limits chan error
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
limit *captureLimitError
|
limit *captureLimitError
|
||||||
stdout *diagnosticWriter
|
stdout *diagnosticWriter
|
||||||
@@ -92,7 +92,7 @@ type diagnosticWriter struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
||||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
logs := &logWriters{limits: make(chan error, 1)}
|
||||||
cleanStdout := cleanLogPath(stdoutPath)
|
cleanStdout := cleanLogPath(stdoutPath)
|
||||||
cleanStderr := cleanLogPath(stderrPath)
|
cleanStderr := cleanLogPath(stderrPath)
|
||||||
|
|
||||||
@@ -221,7 +221,7 @@ func (w *diagnosticWriter) reachLimit() error {
|
|||||||
return limit
|
return limit
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
|
func (l *logWriters) Limits() <-chan error { return l.limits }
|
||||||
|
|
||||||
func (l *logWriters) Limit() *captureLimitError {
|
func (l *logWriters) Limit() *captureLimitError {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
@@ -255,7 +255,7 @@ func openDiagnosticFile(path string) (*os.File, error) {
|
|||||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||||
}
|
}
|
||||||
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,2 +1,4 @@
|
|||||||
// Package subprocess provides reusable process execution and generated-config helpers.
|
// Package subprocess provides internal process execution and Narratio-specific
|
||||||
|
// subprocess policy. Its private execution mechanism owns process lifecycle;
|
||||||
|
// the Run boundary owns environments, diagnostics, redaction, and errors.
|
||||||
package subprocess
|
package subprocess
|
||||||
|
|||||||
91
internal/adapters/subprocess/execution.go
Normal file
91
internal/adapters/subprocess/execution.go
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type executionReason uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
executionNotStarted executionReason = iota
|
||||||
|
executionNaturalExit
|
||||||
|
executionCanceled
|
||||||
|
executionDeadline
|
||||||
|
executionAborted
|
||||||
|
)
|
||||||
|
|
||||||
|
type executionOutcome struct {
|
||||||
|
reason executionReason
|
||||||
|
startedAt time.Time
|
||||||
|
completedAt time.Time
|
||||||
|
exitCode int
|
||||||
|
startErr error
|
||||||
|
waitErr error
|
||||||
|
contextErr error
|
||||||
|
abortErr error
|
||||||
|
cleanupErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeOwnedCommand(ctx context.Context, cmd *exec.Cmd, abort <-chan error) (outcome executionOutcome) {
|
||||||
|
outcome = executionOutcome{startedAt: time.Now().UTC(), exitCode: -1}
|
||||||
|
defer func() { outcome.completedAt = time.Now().UTC() }()
|
||||||
|
if outcome.contextErr = ctx.Err(); outcome.contextErr != nil {
|
||||||
|
outcome.reason = executionNotStarted
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
tree, err := newOwnedProcessTree()
|
||||||
|
if err != nil {
|
||||||
|
outcome.reason, outcome.startErr = executionNotStarted, err
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
defer func() { outcome.cleanupErr = joinErrors(outcome.cleanupErr, tree.Dispose()) }()
|
||||||
|
cmd.WaitDelay = forcefulTerminationWait
|
||||||
|
if outcome.startErr = tree.Start(cmd); outcome.startErr != nil {
|
||||||
|
outcome.reason = executionNotStarted
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
waitCh := make(chan error, 1)
|
||||||
|
go func() { waitCh <- cmd.Wait() }()
|
||||||
|
select {
|
||||||
|
case outcome.waitErr = <-waitCh:
|
||||||
|
outcome.reason = executionNaturalExit
|
||||||
|
case <-ctx.Done():
|
||||||
|
outcome.contextErr = ctx.Err()
|
||||||
|
if outcome.contextErr == context.DeadlineExceeded {
|
||||||
|
outcome.reason = executionDeadline
|
||||||
|
} else {
|
||||||
|
outcome.reason = executionCanceled
|
||||||
|
}
|
||||||
|
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||||
|
case outcome.abortErr = <-abort:
|
||||||
|
outcome.reason = executionAborted
|
||||||
|
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||||
|
}
|
||||||
|
if cmd.ProcessState != nil {
|
||||||
|
outcome.exitCode = cmd.ProcessState.ExitCode()
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
func terminateAndReap(tree ownedProcessTree, waitCh <-chan error) (waitErr, cleanupErr error) {
|
||||||
|
cleanupErr = tree.TerminateGracefully()
|
||||||
|
timer := time.NewTimer(gracefulTerminationWait)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case waitErr = <-waitCh:
|
||||||
|
return waitErr, joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||||
|
case <-timer.C:
|
||||||
|
}
|
||||||
|
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||||
|
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||||
|
defer forcefulTimer.Stop()
|
||||||
|
select {
|
||||||
|
case waitErr = <-waitCh:
|
||||||
|
return waitErr, cleanupErr
|
||||||
|
case <-forcefulTimer.C:
|
||||||
|
return nil, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||||
|
}
|
||||||
|
}
|
||||||
54
internal/adapters/subprocess/execution_test.go
Normal file
54
internal/adapters/subprocess/execution_test.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os/exec"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTerminateAndReapPreservesWaitAndCleanupErrors(t *testing.T) {
|
||||||
|
waitErr := errors.New("wait failed")
|
||||||
|
gracefulErr := errors.New("graceful termination failed")
|
||||||
|
forcefulErr := errors.New("forceful termination failed")
|
||||||
|
tree := &recordingProcessTree{
|
||||||
|
gracefulErr: gracefulErr,
|
||||||
|
forcefulErr: forcefulErr,
|
||||||
|
}
|
||||||
|
waitCh := make(chan error, 1)
|
||||||
|
waitCh <- waitErr
|
||||||
|
|
||||||
|
gotWaitErr, gotCleanupErr := terminateAndReap(tree, waitCh)
|
||||||
|
if !errors.Is(gotWaitErr, waitErr) {
|
||||||
|
t.Fatalf("wait error = %v, want %v", gotWaitErr, waitErr)
|
||||||
|
}
|
||||||
|
for _, want := range []error{gracefulErr, forcefulErr} {
|
||||||
|
if !errors.Is(gotCleanupErr, want) {
|
||||||
|
t.Fatalf("cleanup error = %v, want wrapped %v", gotCleanupErr, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tree.gracefulCalls != 1 || tree.forcefulCalls != 1 {
|
||||||
|
t.Fatalf("termination calls = graceful %d, forceful %d; want 1 each", tree.gracefulCalls, tree.forcefulCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingProcessTree struct {
|
||||||
|
gracefulErr error
|
||||||
|
forcefulErr error
|
||||||
|
|
||||||
|
gracefulCalls int
|
||||||
|
forcefulCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*recordingProcessTree) Start(*exec.Cmd) error { return nil }
|
||||||
|
|
||||||
|
func (tree *recordingProcessTree) TerminateGracefully() error {
|
||||||
|
tree.gracefulCalls++
|
||||||
|
return tree.gracefulErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *recordingProcessTree) TerminateForcefully() error {
|
||||||
|
tree.forcefulCalls++
|
||||||
|
return tree.forcefulErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*recordingProcessTree) Dispose() error { return nil }
|
||||||
@@ -1,9 +1,7 @@
|
|||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -22,39 +20,6 @@ type ownedProcessTree interface {
|
|||||||
Dispose() error
|
Dispose() error
|
||||||
}
|
}
|
||||||
|
|
||||||
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
|
|
||||||
select {
|
|
||||||
case waitErr = <-waitCh:
|
|
||||||
return waitErr, nil, nil, nil
|
|
||||||
case <-ctx.Done():
|
|
||||||
ctxErr = ctx.Err()
|
|
||||||
case captureLimit = <-captureLimits:
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanupErr = tree.TerminateGracefully()
|
|
||||||
gracefulTimer := time.NewTimer(gracefulTerminationWait)
|
|
||||||
defer gracefulTimer.Stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case waitErr = <-waitCh:
|
|
||||||
// The leader may exit before descendants finish graceful shutdown.
|
|
||||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
|
||||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
|
||||||
case <-gracefulTimer.C:
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
|
||||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
|
||||||
defer forcefulTimer.Stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case waitErr = <-waitCh:
|
|
||||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
|
||||||
case <-forcefulTimer.C:
|
|
||||||
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func joinErrors(errs ...error) error {
|
func joinErrors(errs ...error) error {
|
||||||
filtered := make([]error, 0, len(errs))
|
filtered := make([]error, 0, len(errs))
|
||||||
for _, err := range errs {
|
for _, err := range errs {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//go:build linux || darwin || windows
|
//go:build linux || darwin
|
||||||
|
|
||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
//go:build !linux && !darwin && !windows
|
//go:build !linux && !darwin
|
||||||
|
|
||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"runtime"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||||
|
|||||||
@@ -1,122 +0,0 @@
|
|||||||
//go:build windows
|
|
||||||
|
|
||||||
package subprocess
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"syscall"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"golang.org/x/sys/windows"
|
|
||||||
)
|
|
||||||
|
|
||||||
type windowsProcessTree struct {
|
|
||||||
job windows.Handle
|
|
||||||
}
|
|
||||||
|
|
||||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
|
||||||
return &windowsProcessTree{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
|
||||||
job, err := windows.CreateJobObject(nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create job object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
|
||||||
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
||||||
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
|
||||||
_ = windows.CloseHandle(job)
|
|
||||||
return fmt.Errorf("configure job object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
_ = windows.CloseHandle(job)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
|
|
||||||
if err == nil {
|
|
||||||
err = windows.AssignProcessToJobObject(job, process)
|
|
||||||
_ = windows.CloseHandle(process)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = resumeInitialThread(uint32(cmd.Process.Pid))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
killErr := cmd.Process.Kill()
|
|
||||||
waitErr := cmd.Wait()
|
|
||||||
_ = windows.CloseHandle(job)
|
|
||||||
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
tree.job = job
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func resumeInitialThread(processID uint32) error {
|
|
||||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("snapshot initial thread: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = windows.CloseHandle(snapshot) }()
|
|
||||||
|
|
||||||
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
|
||||||
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
|
||||||
return fmt.Errorf("find initial thread: %w", err)
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
if entry.OwnerProcessID != processID {
|
|
||||||
// Keep enumerating until the suspended process's only initial thread
|
|
||||||
// is found.
|
|
||||||
} else {
|
|
||||||
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
|
||||||
if openErr != nil {
|
|
||||||
return fmt.Errorf("open initial thread: %w", openErr)
|
|
||||||
}
|
|
||||||
defer func() { _ = windows.CloseHandle(thread) }()
|
|
||||||
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
|
||||||
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
|
||||||
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return fmt.Errorf("find initial thread: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) TerminateGracefully() error {
|
|
||||||
// Windows jobs have no portable graceful signal. Terminating the owned job
|
|
||||||
// is the safe fallback and prevents a descendant from escaping cleanup.
|
|
||||||
return tree.terminate()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) TerminateForcefully() error {
|
|
||||||
return tree.terminate()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) Dispose() error {
|
|
||||||
if tree.job == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
err := windows.CloseHandle(tree.job)
|
|
||||||
tree.job = 0
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) terminate() error {
|
|
||||||
if tree.job == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return windows.TerminateJobObject(tree.job, 1)
|
|
||||||
}
|
|
||||||
@@ -7,9 +7,6 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// RunRequest defines a subprocess invocation.
|
// RunRequest defines a subprocess invocation.
|
||||||
@@ -60,63 +57,23 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
|||||||
}
|
}
|
||||||
defer logs.Close()
|
defer logs.Close()
|
||||||
|
|
||||||
tree, err := newOwnedProcessTree()
|
|
||||||
if err != nil {
|
|
||||||
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd := exec.Command(req.Executable, req.Args...)
|
cmd := exec.Command(req.Executable, req.Args...)
|
||||||
cmd.Dir = req.WorkingDir
|
cmd.Dir = req.WorkingDir
|
||||||
cmd.Env = childEnv
|
cmd.Env = childEnv
|
||||||
cmd.Stdout = logs.Stdout
|
cmd.Stdout = logs.Stdout
|
||||||
cmd.Stderr = logs.Stderr
|
cmd.Stderr = logs.Stderr
|
||||||
// Streaming capture uses pipes. Bound their lifetime when a leader exits
|
outcome := executeOwnedCommand(runCtx, cmd, logs.Limits())
|
||||||
// while a descendant still holds a stream descriptor.
|
cleanupErr := joinErrors(outcome.cleanupErr, logs.Flush())
|
||||||
cmd.WaitDelay = forcefulTerminationWait
|
captureLimit, _ := outcome.abortErr.(*captureLimitError)
|
||||||
|
|
||||||
started := time.Now().UTC()
|
|
||||||
result := RunResult{
|
|
||||||
ExitCode: -1,
|
|
||||||
StartedAt: started,
|
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
|
||||||
StderrLogPath: req.StderrLogPath,
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := runCtx.Err(); err != nil {
|
|
||||||
result.CompletedAt = time.Now().UTC()
|
|
||||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
|
||||||
return result, fmt.Errorf("command was not started: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tree.Start(cmd); err != nil {
|
|
||||||
result.CompletedAt = time.Now().UTC()
|
|
||||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
|
||||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
waitCh := make(chan error, 1)
|
|
||||||
go func() { waitCh <- cmd.Wait() }()
|
|
||||||
|
|
||||||
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
|
|
||||||
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
|
|
||||||
cleanupErr = joinErrors(cleanupErr, logs.Flush())
|
|
||||||
if captureLimit == nil {
|
if captureLimit == nil {
|
||||||
captureLimit = logs.Limit()
|
captureLimit = logs.Limit()
|
||||||
}
|
}
|
||||||
result.CompletedAt = time.Now().UTC()
|
result := RunResult{ExitCode: outcome.exitCode, StartedAt: outcome.startedAt, CompletedAt: outcome.completedAt, StdoutLogPath: req.StdoutLogPath, StderrLogPath: req.StderrLogPath}
|
||||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||||
if cmd.ProcessState != nil {
|
result.TimedOut = outcome.reason == executionDeadline
|
||||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
result.Canceled = outcome.reason == executionCanceled
|
||||||
}
|
|
||||||
|
|
||||||
if ctxErr == context.DeadlineExceeded {
|
if outcome.reason == executionNaturalExit && outcome.waitErr == nil && cleanupErr == nil {
|
||||||
result.TimedOut = true
|
|
||||||
}
|
|
||||||
if ctxErr == context.Canceled && !result.TimedOut {
|
|
||||||
result.Canceled = true
|
|
||||||
}
|
|
||||||
|
|
||||||
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,48 +81,28 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
|||||||
diagnostics := buildDiagnostics(req, result, stderrTail)
|
diagnostics := buildDiagnostics(req, result, stderrTail)
|
||||||
|
|
||||||
if captureLimit != nil {
|
if captureLimit != nil {
|
||||||
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
|
if cause := joinErrors(outcome.waitErr, cleanupErr); cause != nil {
|
||||||
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
||||||
}
|
}
|
||||||
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
||||||
}
|
}
|
||||||
if result.TimedOut {
|
if result.TimedOut {
|
||||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
if result.Canceled {
|
if result.Canceled {
|
||||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
if exitErr, ok := outcome.waitErr.(*exec.ExitError); ok {
|
||||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
|
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
if cleanupErr != nil {
|
if cleanupErr != nil {
|
||||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
if outcome.startErr != nil {
|
||||||
}
|
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, outcome.startErr)
|
||||||
|
|
||||||
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
|
||||||
func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
|
|
||||||
data, err := yaml.Marshal(value)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshal yaml for %q: %w", path, err)
|
|
||||||
}
|
}
|
||||||
if err := WriteFileAtomic(path, data, perm); err != nil {
|
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(outcome.waitErr, outcome.contextErr, outcome.abortErr, cleanupErr))
|
||||||
return fmt.Errorf("write yaml %q: %w", path, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
|
|
||||||
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
|
||||||
if strings.TrimSpace(path) == "" {
|
|
||||||
return fmt.Errorf("write file: path is required")
|
|
||||||
}
|
|
||||||
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
|
||||||
return fmt.Errorf("write file %q: %w", path, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
||||||
|
|||||||
@@ -8,14 +8,11 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRunSuccessCapturesStdoutStderr(t *testing.T) {
|
func TestRunSuccessCapturesStdoutStderr(t *testing.T) {
|
||||||
@@ -139,9 +136,6 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
|
||||||
}
|
|
||||||
exe, err := os.Executable()
|
exe, err := os.Executable()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("os.Executable() error = %v", err)
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
@@ -165,15 +159,15 @@ func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
|||||||
StdoutLogPath: stdoutPath,
|
StdoutLogPath: stdoutPath,
|
||||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
data, readErr := os.ReadFile(targetPath)
|
data, readErr := os.ReadFile(targetPath)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
t.Fatalf("ReadFile(target) error = %v", readErr)
|
t.Fatalf("ReadFile(target) error = %v", readErr)
|
||||||
}
|
}
|
||||||
if string(data) != original {
|
if string(data) == original {
|
||||||
t.Fatalf("target content = %q, want %q", data, original)
|
t.Fatal("diagnostic target was not replaced")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,7 +376,7 @@ func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||||
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
exactLogs := &logWriters{limits: make(chan error, 1)}
|
||||||
var exactOutput bytes.Buffer
|
var exactOutput bytes.Buffer
|
||||||
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
||||||
if _, err := exact.Write([]byte("abcde")); err != nil {
|
if _, err := exact.Write([]byte("abcde")); err != nil {
|
||||||
@@ -398,7 +392,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
|||||||
t.Fatal("exact write recorded a capture limit")
|
t.Fatal("exact write recorded a capture limit")
|
||||||
}
|
}
|
||||||
|
|
||||||
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
cappedLogs := &logWriters{limits: make(chan error, 1)}
|
||||||
var cappedOutput bytes.Buffer
|
var cappedOutput bytes.Buffer
|
||||||
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
||||||
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
||||||
@@ -416,7 +410,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
||||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
logs := &logWriters{limits: make(chan error, 1)}
|
||||||
var output bytes.Buffer
|
var output bytes.Buffer
|
||||||
secret := "credential-value"
|
secret := "credential-value"
|
||||||
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
||||||
@@ -592,59 +586,6 @@ func TestRunSharedStdoutStderrLogPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteYAMLAtomic(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "config.generated.yml")
|
|
||||||
|
|
||||||
if err := WriteYAMLAtomic(path, map[string]any{"name": "narratio", "stage": "merge"}, 0o644); err != nil {
|
|
||||||
t.Fatalf("WriteYAMLAtomic() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read yaml: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
if err := yaml.Unmarshal(data, &got); err != nil {
|
|
||||||
t.Fatalf("yaml unmarshal: %v", err)
|
|
||||||
}
|
|
||||||
if got["name"] != "narratio" {
|
|
||||||
t.Fatalf("name = %#v, want narratio", got["name"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteYAMLAtomicOverwriteNoTempResidue(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "config.generated.yml")
|
|
||||||
|
|
||||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "one"}, 0o644); err != nil {
|
|
||||||
t.Fatalf("first write: %v", err)
|
|
||||||
}
|
|
||||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "two"}, 0o644); err != nil {
|
|
||||||
t.Fatalf("second write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read yaml: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), "two") {
|
|
||||||
t.Fatalf("yaml = %q, want overwritten value", string(data))
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadDir() error = %v", err)
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
name := entry.Name()
|
|
||||||
if strings.Contains(name, ".tmp-") {
|
|
||||||
t.Fatalf("temp file residue found: %q", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSubprocessHelper(t *testing.T) {
|
func TestSubprocessHelper(t *testing.T) {
|
||||||
if os.Getenv("GO_WANT_SUBPROCESS_HELPER") != "1" {
|
if os.Getenv("GO_WANT_SUBPROCESS_HELPER") != "1" {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -6,14 +6,11 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
@@ -21,7 +18,6 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
||||||
whisperXUploadBufferSize = 32 * 1024
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
||||||
@@ -192,21 +188,32 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
||||||
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
|
upload, err := newMultipartBodyFactory(ctx, audioPath, c.language, c.openAudio)
|
||||||
defer upload.Close()
|
if err != nil {
|
||||||
|
return 0, nil, fmt.Errorf("prepare whisperx request body: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = upload.Close() }()
|
||||||
|
body, err := upload.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
return 0, nil, fmt.Errorf("open whisperx request body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", upload.contentType)
|
req.Header.Set("Content-Type", upload.contentType)
|
||||||
req.Header.Set("Expect", "100-continue")
|
req.Header.Set("Expect", "100-continue")
|
||||||
|
req.GetBody = upload.NewBody
|
||||||
|
|
||||||
resp, err := c.httpClient.Do(req)
|
resp, err := c.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = upload.Close()
|
_ = upload.Close()
|
||||||
if producerErr := upload.Wait(); producerErr != nil {
|
if sourceErr := upload.SourceError(true); sourceErr != nil {
|
||||||
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
return 0, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return 0, nil, fmt.Errorf("perform whisperx request: %w", ctxErr)
|
||||||
}
|
}
|
||||||
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
@@ -214,8 +221,8 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
|
|||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
_ = upload.Close()
|
_ = upload.Close()
|
||||||
if producerErr := upload.Wait(); producerErr != nil {
|
if sourceErr := upload.SourceError(false); sourceErr != nil {
|
||||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
}
|
}
|
||||||
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
||||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||||
@@ -223,13 +230,19 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
|
|||||||
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := upload.Wait(); err != nil {
|
data, responseErr := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
||||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
|
_ = upload.Close()
|
||||||
|
if sourceErr := upload.SourceError(false); sourceErr != nil {
|
||||||
|
if responseErr != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", errors.Join(sourceErr, responseErr))
|
||||||
|
}
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
return resp.StatusCode, nil, fmt.Errorf("complete whisperx request: %w", err)
|
||||||
if err != nil {
|
}
|
||||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
if responseErr != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", responseErr)
|
||||||
}
|
}
|
||||||
return resp.StatusCode, data, nil
|
return resp.StatusCode, data, nil
|
||||||
}
|
}
|
||||||
@@ -286,161 +299,6 @@ func isHTTPURLScheme(scheme string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type multipartUpload struct {
|
|
||||||
reader *io.PipeReader
|
|
||||||
writer *io.PipeWriter
|
|
||||||
contentType string
|
|
||||||
done chan struct{}
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
audio io.Closer
|
|
||||||
err error
|
|
||||||
aborted bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
|
|
||||||
reader, writer := io.Pipe()
|
|
||||||
multipartWriter := multipart.NewWriter(writer)
|
|
||||||
upload := &multipartUpload{
|
|
||||||
reader: reader,
|
|
||||||
writer: writer,
|
|
||||||
contentType: multipartWriter.FormDataContentType(),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
|
|
||||||
if err != nil {
|
|
||||||
_ = writer.CloseWithError(err)
|
|
||||||
} else {
|
|
||||||
_ = writer.Close()
|
|
||||||
}
|
|
||||||
upload.mu.Lock()
|
|
||||||
upload.err = err
|
|
||||||
upload.audio = nil
|
|
||||||
upload.mu.Unlock()
|
|
||||||
close(upload.done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
upload.abort()
|
|
||||||
case <-upload.done:
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return upload
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Read(p []byte) (int, error) {
|
|
||||||
return u.reader.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Close() error {
|
|
||||||
u.abort()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Wait() error {
|
|
||||||
<-u.done
|
|
||||||
u.mu.Lock()
|
|
||||||
defer u.mu.Unlock()
|
|
||||||
return u.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
|
|
||||||
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
|
||||||
if err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
audioFile, err := openAudio(audioPath)
|
|
||||||
if err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
|
|
||||||
}
|
|
||||||
u.setAudio(audioFile)
|
|
||||||
|
|
||||||
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
|
|
||||||
closeErr := audioFile.Close()
|
|
||||||
u.clearAudio(audioFile)
|
|
||||||
if copyErr != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := writer.WriteField("language", language); err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
|
|
||||||
}
|
|
||||||
if err := writer.Close(); err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
u.mu.Lock()
|
|
||||||
aborted := u.aborted
|
|
||||||
u.mu.Unlock()
|
|
||||||
if aborted {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) setAudio(audio io.Closer) {
|
|
||||||
u.mu.Lock()
|
|
||||||
u.audio = audio
|
|
||||||
aborted := u.aborted
|
|
||||||
u.mu.Unlock()
|
|
||||||
if aborted {
|
|
||||||
_ = audio.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) clearAudio(audio io.Closer) {
|
|
||||||
u.mu.Lock()
|
|
||||||
if u.audio == audio {
|
|
||||||
u.audio = nil
|
|
||||||
}
|
|
||||||
u.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) abort() {
|
|
||||||
u.mu.Lock()
|
|
||||||
if u.aborted {
|
|
||||||
u.mu.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u.aborted = true
|
|
||||||
audio := u.audio
|
|
||||||
u.mu.Unlock()
|
|
||||||
|
|
||||||
_ = u.reader.Close()
|
|
||||||
if audio != nil {
|
|
||||||
_ = audio.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type contextReader struct {
|
|
||||||
ctx context.Context
|
|
||||||
reader io.Reader
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *contextReader) Read(p []byte) (int, error) {
|
|
||||||
select {
|
|
||||||
case <-r.ctx.Done():
|
|
||||||
return 0, r.ctx.Err()
|
|
||||||
default:
|
|
||||||
return r.reader.Read(p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||||
if strings.TrimSpace(path) == "" {
|
if strings.TrimSpace(path) == "" {
|
||||||
return fmt.Errorf("path is required")
|
return fmt.Errorf("path is required")
|
||||||
|
|||||||
261
internal/adapters/whisperx/http_replay_test.go
Normal file
261
internal/adapters/whisperx/http_replay_test.go
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHTTPClientTransportReplaysAbandonedMultipartBody(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
partial := make([]byte, 17)
|
||||||
|
if _, err := io.ReadFull(request.Body, partial); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := request.Body.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if request.GetBody == nil {
|
||||||
|
return nil, errors.New("request GetBody is nil")
|
||||||
|
}
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
replayedBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
_ = replay.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := replay.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parts := parseMultipartParts(t, request.Header.Get("Content-Type"), replayedBytes)
|
||||||
|
if len(parts) != 2 || parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
|
||||||
|
t.Fatalf("replayed file part = %#v, want exact audio.flac payload", parts)
|
||||||
|
}
|
||||||
|
if parts[1].name != "language" || string(parts[1].data) != "en" {
|
||||||
|
t.Fatalf("replayed language part = %#v, want language=en", parts[1])
|
||||||
|
}
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
opens.Add(1)
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
result, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Attempts != 1 {
|
||||||
|
t.Fatalf("application attempts = %d, want 1", result.Attempts)
|
||||||
|
}
|
||||||
|
verifyJSONFile(t, outPath)
|
||||||
|
if got := opens.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio opens = %d, want original and replay", got)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio closes = %d, want all body handles disposed", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportReturnsReplayOpenFailure(t *testing.T) {
|
||||||
|
replayErr := errors.New("replay open failed")
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
_, err := request.GetBody()
|
||||||
|
return nil, err
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 2 {
|
||||||
|
return nil, replayErr
|
||||||
|
}
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
result, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, replayErr) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want replay open failure", err)
|
||||||
|
}
|
||||||
|
if result.Attempts != 1 {
|
||||||
|
t.Fatalf("application attempts = %d, want 1", result.Attempts)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 1 {
|
||||||
|
t.Fatalf("audio closes = %d, want original disposed", got)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
|
t.Fatalf("output should not exist, stat error = %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportCannotSwallowReplayReadFailure(t *testing.T) {
|
||||||
|
replayErr := errors.New("replay read failed")
|
||||||
|
var opens atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_, _ = io.ReadAll(replay)
|
||||||
|
_ = replay.Close()
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 2 {
|
||||||
|
return &failingReadCloser{first: []byte("partial"), err: replayErr}, nil
|
||||||
|
}
|
||||||
|
return io.NopCloser(strings.NewReader("audio-data")), nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, replayErr) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want replay read failure", err)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
|
t.Fatalf("output should not exist, stat error = %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportEarlyResponseDisposesOriginalBody(t *testing.T) {
|
||||||
|
var closes atomic.Int32
|
||||||
|
client := newReplayTestHTTPClient(t, roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
}))
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 1 {
|
||||||
|
t.Fatalf("audio closes = %d, want early-response body disposed", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientParentCancellationDuringReplayClosesEveryBody(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var originalCloses atomic.Int32
|
||||||
|
replaySource := newSignalingBlockedReadCloser()
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_, _ = io.ReadAll(replay)
|
||||||
|
_ = replay.Close()
|
||||||
|
return nil, request.Context().Err()
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 1 {
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &originalCloses}, nil
|
||||||
|
}
|
||||||
|
return replaySource, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := client.Transcribe(ctx, TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
|
||||||
|
})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-replaySource.blocked:
|
||||||
|
cancel()
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
cancel()
|
||||||
|
t.Fatal("replay source did not block")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Transcribe() did not return after replay cancellation")
|
||||||
|
}
|
||||||
|
if got := originalCloses.Load(); got != 1 {
|
||||||
|
t.Fatalf("original audio closes = %d, want 1", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-replaySource.closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("replay audio source was not closed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (roundTrip roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
return roundTrip(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newReplayTestHTTPClient(t *testing.T, transport http.RoundTripper) *HTTPClient {
|
||||||
|
t.Helper()
|
||||||
|
client, err := NewHTTPClient(HTTPClientConfig{
|
||||||
|
TranscribeURL: "http://whisperx.test/transcribe",
|
||||||
|
Language: "en",
|
||||||
|
Timeout: time.Second,
|
||||||
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewHTTPClient() error = %v", err)
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
func whisperXJSONResponse(request *http.Request) *http.Response {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Status: "200 OK",
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: io.NopCloser(strings.NewReader(`{"source":"transport-replay","segments":[]}`)),
|
||||||
|
Request: request,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -365,7 +365,7 @@ func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
|
func TestHTTPClientCancellationReleasesBlockedSource(t *testing.T) {
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
source := newGatedReadCloser([]byte("audio-data"), release)
|
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||||
firstByteReceived := make(chan struct{})
|
firstByteReceived := make(chan struct{})
|
||||||
|
|||||||
263
internal/adapters/whisperx/multipart_body.go
Normal file
263
internal/adapters/whisperx/multipart_body.go
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
type multipartBodyFactory struct {
|
||||||
|
ctx context.Context
|
||||||
|
audioPath string
|
||||||
|
openAudio func(string) (io.ReadCloser, error)
|
||||||
|
prefix []byte
|
||||||
|
suffix []byte
|
||||||
|
contentType string
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
bodies []*multipartBody
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMultipartBodyFactory(
|
||||||
|
ctx context.Context,
|
||||||
|
audioPath string,
|
||||||
|
language string,
|
||||||
|
openAudio func(string) (io.ReadCloser, error),
|
||||||
|
) (*multipartBodyFactory, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
if openAudio == nil {
|
||||||
|
return nil, fmt.Errorf("audio opener is required")
|
||||||
|
}
|
||||||
|
var framing bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&framing)
|
||||||
|
if _, err := writer.CreateFormFile("file", filepath.Base(audioPath)); err != nil {
|
||||||
|
return nil, fmt.Errorf("create multipart file field: %w", err)
|
||||||
|
}
|
||||||
|
prefixLength := framing.Len()
|
||||||
|
if err := writer.WriteField("language", language); err != nil {
|
||||||
|
return nil, fmt.Errorf("write language form field: %w", err)
|
||||||
|
}
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
return nil, fmt.Errorf("close multipart framing: %w", err)
|
||||||
|
}
|
||||||
|
framingBytes := framing.Bytes()
|
||||||
|
return &multipartBodyFactory{
|
||||||
|
ctx: ctx,
|
||||||
|
audioPath: audioPath,
|
||||||
|
openAudio: openAudio,
|
||||||
|
prefix: append([]byte(nil), framingBytes[:prefixLength]...),
|
||||||
|
suffix: append([]byte(nil), framingBytes[prefixLength:]...),
|
||||||
|
contentType: writer.FormDataContentType(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) NewBody() (io.ReadCloser, error) {
|
||||||
|
factory.mu.Lock()
|
||||||
|
if factory.closed {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, fmt.Errorf("multipart body factory is closed")
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
audio, err := factory.openAudio(factory.audioPath)
|
||||||
|
if err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, fmt.Errorf("open audio file %q: %w", factory.audioPath, err)
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
_ = audio.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
source := &multipartAudioSource{audio: audio, closeDone: make(chan struct{})}
|
||||||
|
body := &multipartBody{
|
||||||
|
ctx: factory.ctx,
|
||||||
|
source: source,
|
||||||
|
reader: io.MultiReader(bytes.NewReader(factory.prefix), source, bytes.NewReader(factory.suffix)),
|
||||||
|
}
|
||||||
|
factory.bodies = append(factory.bodies, body)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
stopCancel := context.AfterFunc(factory.ctx, func() {
|
||||||
|
_ = body.Close()
|
||||||
|
})
|
||||||
|
body.mu.Lock()
|
||||||
|
body.stopCancel = stopCancel
|
||||||
|
closed := body.closed
|
||||||
|
body.mu.Unlock()
|
||||||
|
if closed {
|
||||||
|
stopCancel()
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
_ = body.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) Close() error {
|
||||||
|
factory.mu.Lock()
|
||||||
|
if factory.closed {
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return multipartBodyCloseErrors(bodies)
|
||||||
|
}
|
||||||
|
factory.closed = true
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
for _, body := range bodies {
|
||||||
|
_ = body.Close()
|
||||||
|
}
|
||||||
|
return multipartBodyCloseErrors(bodies)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) SourceError(includeAbandoned bool) error {
|
||||||
|
factory.mu.Lock()
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
var result error
|
||||||
|
for _, body := range bodies {
|
||||||
|
if err := body.sourceError(includeAbandoned); err != nil {
|
||||||
|
result = errors.Join(result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func multipartBodyCloseErrors(bodies []*multipartBody) error {
|
||||||
|
var result error
|
||||||
|
for _, body := range bodies {
|
||||||
|
if err := body.sourceError(false); err != nil {
|
||||||
|
result = errors.Join(result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartBody struct {
|
||||||
|
ctx context.Context
|
||||||
|
reader io.Reader
|
||||||
|
source *multipartAudioSource
|
||||||
|
stopCancel func() bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
abandoned bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) Read(buffer []byte) (int, error) {
|
||||||
|
if err := body.ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
body.mu.Lock()
|
||||||
|
closed := body.closed
|
||||||
|
body.mu.Unlock()
|
||||||
|
if closed {
|
||||||
|
return 0, io.ErrClosedPipe
|
||||||
|
}
|
||||||
|
return body.reader.Read(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) Close() error {
|
||||||
|
body.mu.Lock()
|
||||||
|
if body.closed {
|
||||||
|
body.mu.Unlock()
|
||||||
|
return body.source.Close()
|
||||||
|
}
|
||||||
|
body.closed = true
|
||||||
|
body.abandoned = !body.source.reachedEOF()
|
||||||
|
stopCancel := body.stopCancel
|
||||||
|
body.mu.Unlock()
|
||||||
|
if stopCancel != nil {
|
||||||
|
stopCancel()
|
||||||
|
}
|
||||||
|
return body.source.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) sourceError(includeAbandoned bool) error {
|
||||||
|
body.mu.Lock()
|
||||||
|
abandoned := body.abandoned
|
||||||
|
body.mu.Unlock()
|
||||||
|
err := body.source.Err()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if abandoned && !includeAbandoned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartAudioSource struct {
|
||||||
|
audio io.ReadCloser
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closing bool
|
||||||
|
eof bool
|
||||||
|
readErr error
|
||||||
|
closeErr error
|
||||||
|
closeDone chan struct{}
|
||||||
|
closeResult error
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Read(buffer []byte) (int, error) {
|
||||||
|
n, err := source.audio.Read(buffer)
|
||||||
|
source.mu.Lock()
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, io.EOF):
|
||||||
|
source.eof = true
|
||||||
|
case err != nil && !source.closing:
|
||||||
|
source.readErr = errors.Join(source.readErr, err)
|
||||||
|
}
|
||||||
|
source.mu.Unlock()
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Close() error {
|
||||||
|
source.mu.Lock()
|
||||||
|
if source.closed {
|
||||||
|
done := source.closeDone
|
||||||
|
source.mu.Unlock()
|
||||||
|
<-done
|
||||||
|
source.mu.Lock()
|
||||||
|
err := source.closeResult
|
||||||
|
source.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source.closed = true
|
||||||
|
source.closing = true
|
||||||
|
reachedEOF := source.eof
|
||||||
|
source.mu.Unlock()
|
||||||
|
|
||||||
|
err := source.audio.Close()
|
||||||
|
source.mu.Lock()
|
||||||
|
source.closeResult = err
|
||||||
|
if err != nil && reachedEOF {
|
||||||
|
source.closeErr = err
|
||||||
|
}
|
||||||
|
close(source.closeDone)
|
||||||
|
source.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) reachedEOF() bool {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return source.eof
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Err() error {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return errors.Join(source.readErr, source.closeErr)
|
||||||
|
}
|
||||||
327
internal/adapters/whisperx/multipart_body_test.go
Normal file
327
internal/adapters/whisperx/multipart_body_test.go
Normal file
@@ -0,0 +1,327 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"mime/multipart"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMultipartBodyReplayUsesIdenticalFraming(t *testing.T) {
|
||||||
|
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
|
||||||
|
defer factory.Close()
|
||||||
|
|
||||||
|
original, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() original error = %v", err)
|
||||||
|
}
|
||||||
|
replay, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() replay error = %v", err)
|
||||||
|
}
|
||||||
|
originalBytes, err := io.ReadAll(original)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(original) error = %v", err)
|
||||||
|
}
|
||||||
|
replayBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(replay) error = %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(originalBytes, replayBytes) {
|
||||||
|
t.Fatal("original and replay multipart bytes differ")
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := parseMultipartParts(t, factory.contentType, replayBytes)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
t.Fatalf("part count = %d, want 2", len(parts))
|
||||||
|
}
|
||||||
|
if parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
|
||||||
|
t.Fatalf("file part = %#v, want file/audio.flac with exact audio", parts[0])
|
||||||
|
}
|
||||||
|
if parts[1].name != "language" || string(parts[1].data) != "en" {
|
||||||
|
t.Fatalf("language part = %#v, want language=en", parts[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyReplayRestartsAfterPartialOriginal(t *testing.T) {
|
||||||
|
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
|
||||||
|
defer factory.Close()
|
||||||
|
|
||||||
|
original, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() original error = %v", err)
|
||||||
|
}
|
||||||
|
partial := make([]byte, len(factory.prefix)+3)
|
||||||
|
if _, err := io.ReadFull(original, partial); err != nil {
|
||||||
|
t.Fatalf("ReadFull(original) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := original.Close(); err != nil {
|
||||||
|
t.Fatalf("Close(original) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
replay, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() replay error = %v", err)
|
||||||
|
}
|
||||||
|
replayBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(replay) error = %v", err)
|
||||||
|
}
|
||||||
|
parts := parseMultipartParts(t, factory.contentType, replayBytes)
|
||||||
|
if got := string(parts[0].data); got != "audio-data" {
|
||||||
|
t.Fatalf("replayed audio = %q, want complete payload", got)
|
||||||
|
}
|
||||||
|
if err := factory.SourceError(false); err != nil {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want abandoned original ignored", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodySimultaneousBodiesUseIndependentSources(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
opens.Add(1)
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() first error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() second error = %v", err)
|
||||||
|
}
|
||||||
|
var firstBytes, secondBytes []byte
|
||||||
|
var firstErr, secondErr error
|
||||||
|
var group sync.WaitGroup
|
||||||
|
group.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer group.Done()
|
||||||
|
firstBytes, firstErr = io.ReadAll(first)
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer group.Done()
|
||||||
|
secondBytes, secondErr = io.ReadAll(second)
|
||||||
|
}()
|
||||||
|
group.Wait()
|
||||||
|
if firstErr != nil || secondErr != nil {
|
||||||
|
t.Fatalf("concurrent reads returned (%v, %v)", firstErr, secondErr)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(firstBytes, secondBytes) {
|
||||||
|
t.Fatal("independent body bytes differ")
|
||||||
|
}
|
||||||
|
if got := opens.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio opens = %d, want 2", got)
|
||||||
|
}
|
||||||
|
if err := factory.Close(); err != nil {
|
||||||
|
t.Fatalf("Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := factory.Close(); err != nil {
|
||||||
|
t.Fatalf("second Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio closes = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyReportsSourceFailures(t *testing.T) {
|
||||||
|
t.Run("read", func(t *testing.T) {
|
||||||
|
sourceErr := errors.New("read source")
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := io.ReadAll(body); !errors.Is(err, sourceErr) {
|
||||||
|
t.Fatalf("ReadAll() error = %v, want source failure", err)
|
||||||
|
}
|
||||||
|
_ = body.Close()
|
||||||
|
if err := factory.SourceError(false); !errors.Is(err, sourceErr) {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want source failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("close after complete read", func(t *testing.T) {
|
||||||
|
closeErr := errors.New("close source")
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return &errorCloseReadCloser{Reader: strings.NewReader("audio-data"), err: closeErr}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := io.ReadAll(body); err != nil {
|
||||||
|
t.Fatalf("ReadAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := body.Close(); !errors.Is(err, closeErr) {
|
||||||
|
t.Fatalf("Close() error = %v, want source close failure", err)
|
||||||
|
}
|
||||||
|
if err := factory.SourceError(false); !errors.Is(err, closeErr) {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want source close failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyCloseAndCancelReleaseBlockedSource(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
release func(context.CancelFunc, io.ReadCloser)
|
||||||
|
}{
|
||||||
|
{name: "close", release: func(_ context.CancelFunc, body io.ReadCloser) { _ = body.Close() }},
|
||||||
|
{name: "cancel", release: func(cancel context.CancelFunc, _ io.ReadCloser) { cancel() }},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
source := newSignalingBlockedReadCloser()
|
||||||
|
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return source, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
defer factory.Close()
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
readDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := io.ReadAll(body)
|
||||||
|
readDone <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-source.blocked:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("audio source did not block")
|
||||||
|
}
|
||||||
|
test.release(cancel, body)
|
||||||
|
select {
|
||||||
|
case <-source.closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("audio source was not closed")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-readDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("blocked body read did not return")
|
||||||
|
}
|
||||||
|
if test.name == "cancel" {
|
||||||
|
if _, err := factory.NewBody(); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("NewBody() after cancellation error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestMultipartBodyFactory(t *testing.T, ctx context.Context, audio string) *multipartBodyFactory {
|
||||||
|
t.Helper()
|
||||||
|
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return io.NopCloser(strings.NewReader(audio)), nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
return factory
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartTestPart struct {
|
||||||
|
name string
|
||||||
|
filename string
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseMultipartParts(t *testing.T, contentType string, body []byte) []multipartTestPart {
|
||||||
|
t.Helper()
|
||||||
|
mediaType, params, err := mime.ParseMediaType(contentType)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseMediaType() error = %v", err)
|
||||||
|
}
|
||||||
|
if mediaType != "multipart/form-data" {
|
||||||
|
t.Fatalf("media type = %q, want multipart/form-data", mediaType)
|
||||||
|
}
|
||||||
|
reader := multipart.NewReader(bytes.NewReader(body), params["boundary"])
|
||||||
|
var parts []multipartTestPart
|
||||||
|
for {
|
||||||
|
part, err := reader.NextPart()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NextPart() error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(part)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(part %q) error = %v", part.FormName(), err)
|
||||||
|
}
|
||||||
|
parts = append(parts, multipartTestPart{name: part.FormName(), filename: part.FileName(), data: data})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type countedReadCloser struct {
|
||||||
|
io.Reader
|
||||||
|
closes *atomic.Int32
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *countedReadCloser) Close() error {
|
||||||
|
reader.once.Do(func() { reader.closes.Add(1) })
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type errorCloseReadCloser struct {
|
||||||
|
io.Reader
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *errorCloseReadCloser) Close() error { return reader.err }
|
||||||
|
|
||||||
|
type signalingBlockedReadCloser struct {
|
||||||
|
blocked chan struct{}
|
||||||
|
closed chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
read bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSignalingBlockedReadCloser() *signalingBlockedReadCloser {
|
||||||
|
return &signalingBlockedReadCloser{blocked: make(chan struct{}), closed: make(chan struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *signalingBlockedReadCloser) Read(buffer []byte) (int, error) {
|
||||||
|
if !reader.read {
|
||||||
|
reader.read = true
|
||||||
|
return copy(buffer, "audio-data"), nil
|
||||||
|
}
|
||||||
|
reader.once.Do(func() { close(reader.blocked) })
|
||||||
|
<-reader.closed
|
||||||
|
return 0, errors.New("audio source closed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *signalingBlockedReadCloser) Close() error {
|
||||||
|
select {
|
||||||
|
case <-reader.closed:
|
||||||
|
default:
|
||||||
|
close(reader.closed)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -38,11 +38,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
|
|
||||||
var capturedStages []string
|
var capturedStages []string
|
||||||
var capturedArtifacts []string
|
var capturedArtifacts []string
|
||||||
origExecuteStagesFn := executeStagesFn
|
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||||
t.Cleanup(func() {
|
|
||||||
executeStagesFn = origExecuteStagesFn
|
|
||||||
})
|
|
||||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
||||||
for _, s := range plan.Stages() {
|
for _, s := range plan.Stages() {
|
||||||
capturedStages = append(capturedStages, s.Name())
|
capturedStages = append(capturedStages, s.Name())
|
||||||
}
|
}
|
||||||
@@ -52,7 +48,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := executeWithPipelineExecutor(
|
||||||
[]string{
|
[]string{
|
||||||
"run-stage", "publish", "2026-05-03",
|
"run-stage", "publish", "2026-05-03",
|
||||||
"--config", pipelinePath,
|
"--config", pipelinePath,
|
||||||
@@ -62,6 +58,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
},
|
},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
|
executor,
|
||||||
)
|
)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
@@ -154,7 +151,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=4 skipped=8") {
|
if !strings.Contains(out.String(), "executed=5 skipped=9") {
|
||||||
t.Fatalf("output = %q, want extract reconsidered and legacy analyze plus delivery rebuilt", out.String())
|
t.Fatalf("output = %q, want extract reconsidered and legacy analyze plus delivery rebuilt", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -165,11 +162,7 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
|||||||
|
|
||||||
var capturedStages []string
|
var capturedStages []string
|
||||||
var capturedForce bool
|
var capturedForce bool
|
||||||
origExecuteStagesFn := executeStagesFn
|
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||||
t.Cleanup(func() {
|
|
||||||
executeStagesFn = origExecuteStagesFn
|
|
||||||
})
|
|
||||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
||||||
for _, s := range plan.Stages() {
|
for _, s := range plan.Stages() {
|
||||||
capturedStages = append(capturedStages, s.Name())
|
capturedStages = append(capturedStages, s.Name())
|
||||||
}
|
}
|
||||||
@@ -182,10 +175,11 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := executeWithPipelineExecutor(
|
||||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
|
executor,
|
||||||
)
|
)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
@@ -206,18 +200,14 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
var capturedArtifacts []string
|
var capturedArtifacts []string
|
||||||
origExecuteStagesFn := executeStagesFn
|
executor := func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||||
t.Cleanup(func() {
|
|
||||||
executeStagesFn = origExecuteStagesFn
|
|
||||||
})
|
|
||||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
||||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := executeWithPipelineExecutor(
|
||||||
[]string{
|
[]string{
|
||||||
"analyze",
|
"analyze",
|
||||||
"2026-05-03",
|
"2026-05-03",
|
||||||
@@ -228,6 +218,7 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
|||||||
},
|
},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
|
executor,
|
||||||
)
|
)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
@@ -281,13 +272,18 @@ func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
code := executeWithDependencies([]string{"analyze", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "analyze: no pipeline config path provided") ||
|
||||||
|
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -299,11 +295,7 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
|||||||
var capturedStages []string
|
var capturedStages []string
|
||||||
var capturedForce bool
|
var capturedForce bool
|
||||||
var capturedArtifacts []string
|
var capturedArtifacts []string
|
||||||
origExecuteStagesFn := executeStagesFn
|
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||||
t.Cleanup(func() {
|
|
||||||
executeStagesFn = origExecuteStagesFn
|
|
||||||
})
|
|
||||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
||||||
for _, s := range plan.Stages() {
|
for _, s := range plan.Stages() {
|
||||||
capturedStages = append(capturedStages, s.Name())
|
capturedStages = append(capturedStages, s.Name())
|
||||||
}
|
}
|
||||||
@@ -317,10 +309,11 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := executeWithPipelineExecutor(
|
||||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
|
executor,
|
||||||
)
|
)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
@@ -383,13 +376,18 @@ func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
code := executeWithDependencies([]string{"publish", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "publish: no pipeline config path provided") ||
|
||||||
|
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
@@ -0,0 +1,336 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/go-cmp/cmp"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteStagesRetriesFromLastAnalyzeCheckpoint(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: checkpointRecoveryArtifacts("original")}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.Campaign = cfg.Session.Campaign
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim"} {
|
||||||
|
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
}
|
||||||
|
saveBoundedManifest(t, cfg, seed)
|
||||||
|
analyze := mustSelectStage(t, "analyze")
|
||||||
|
|
||||||
|
baselineRunner := &analyzeCheckpointRunner{}
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, Scriptorium: baselineRunner},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("baseline executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
baseline, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load baseline manifest: %v", err)
|
||||||
|
}
|
||||||
|
baseline.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||||
|
baseline.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), baseline); err != nil {
|
||||||
|
t.Fatalf("save delivery baseline: %v", err)
|
||||||
|
}
|
||||||
|
upstream := cloneStageRecords(baseline, "prepare", "transcribe", "merge", "polish", "normalize", "trim")
|
||||||
|
unrelated := baseline.Stages["analyze"].AnalyzeArtifacts["unrelated"]
|
||||||
|
|
||||||
|
cfg.Pipeline.Scriptorium.Artifacts = checkpointRecoveryArtifacts("revised")
|
||||||
|
failure := errors.New("injected quest-log failure")
|
||||||
|
failingRunner := &analyzeCheckpointRunner{FailPrompt: "quest-log-revised", Err: failure}
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, Scriptorium: failingRunner},
|
||||||
|
}); !errors.Is(err, failure) {
|
||||||
|
t.Fatalf("executeStages() error = %v, want quest-log failure", err)
|
||||||
|
}
|
||||||
|
if want := []string{"chronicle-revised", "encounters-revised", "quest-log-revised"}; !slices.Equal(failingRunner.Calls, want) {
|
||||||
|
t.Fatalf("failed-run calls = %#v, want %#v", failingRunner.Calls, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
failed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||||
|
failedAnalyze := failed.Stages["analyze"]
|
||||||
|
if failedAnalyze.Status != manifest.StatusFailed || len(failedAnalyze.Outputs) != 0 {
|
||||||
|
t.Fatalf("aggregate analyze = %#v, want failed without outputs", failedAnalyze)
|
||||||
|
}
|
||||||
|
assertAnalyzeArtifactStatus(t, failedAnalyze, "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||||
|
assertAnalyzeArtifactStatus(t, failedAnalyze, "encounters", manifest.AnalyzeArtifactCurrent)
|
||||||
|
assertAnalyzeArtifactStatus(t, failedAnalyze, "quest_log", manifest.AnalyzeArtifactFailed)
|
||||||
|
assertAnalyzeArtifactStatus(t, failedAnalyze, "player_handout", manifest.AnalyzeArtifactStale)
|
||||||
|
gotUnrelated := failedAnalyze.AnalyzeArtifacts["unrelated"]
|
||||||
|
if diff := cmp.Diff(unrelated, gotUnrelated); diff != "" {
|
||||||
|
t.Fatalf("unrelated artifact changed (-want +got):\n%s", diff)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"publish", "notify"} {
|
||||||
|
if got := failed.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertStageRecordsUnchanged(t, failed, upstream)
|
||||||
|
|
||||||
|
retryRunner := &analyzeCheckpointRunner{}
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, Scriptorium: retryRunner},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"quest-log-revised", "player-handout-revised"}; !slices.Equal(retryRunner.Calls, want) {
|
||||||
|
t.Fatalf("retry calls = %#v, want %#v", retryRunner.Calls, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
completed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||||
|
if completed.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("aggregate analyze = %#v, want succeeded", completed.Stages["analyze"])
|
||||||
|
}
|
||||||
|
for _, key := range []string{"chronicle", "encounters", "player_handout", "quest_log", "unrelated"} {
|
||||||
|
assertAnalyzeArtifactStatus(t, completed.Stages["analyze"], key, manifest.AnalyzeArtifactCurrent)
|
||||||
|
}
|
||||||
|
assertStageRecordsUnchanged(t, completed, upstream)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesJoinsAnalyzeFailureWithTerminalSaveFailure(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
artifactErr := errors.New("injected analyze failure")
|
||||||
|
terminalErr := errors.New("injected terminal save failure")
|
||||||
|
store := &failedAnalyzeSaveStore{Store: &manifest.LocalStore{}, Err: terminalErr}
|
||||||
|
runner := &analyzeCheckpointRunner{FailPrompt: "chronicle", Err: artifactErr}
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, artifactErr) || !errors.Is(err, terminalErr) {
|
||||||
|
t.Fatalf("executeStages() error = %v, want analyze and terminal save failures", err)
|
||||||
|
}
|
||||||
|
if !store.Failed {
|
||||||
|
t.Fatal("terminal save failure was not injected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesPersistsAnalyzeCheckpointFailureRecovery(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||||
|
"encounters": {Enabled: true, PromptID: "encounters", OutputPath: "artifacts/encounters.md"},
|
||||||
|
"quest_log": {Enabled: true, PromptID: "quest-log", OutputPath: "artifacts/quest_log.md"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
baseStore := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.Campaign = cfg.Session.Campaign
|
||||||
|
seed.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||||
|
seed.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||||
|
saveBoundedManifest(t, cfg, seed)
|
||||||
|
checkpointErr := errors.New("injected artifact checkpoint failure")
|
||||||
|
store := ¤tAnalyzeCheckpointFailingStore{
|
||||||
|
Store: baseStore,
|
||||||
|
CurrentLimit: 2,
|
||||||
|
Err: checkpointErr,
|
||||||
|
}
|
||||||
|
runner := &analyzeCheckpointRunner{}
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, checkpointErr) {
|
||||||
|
t.Fatalf("executeStages() error = %v, want checkpoint failure", err)
|
||||||
|
}
|
||||||
|
if want := []string{"chronicle", "encounters"}; !slices.Equal(runner.Calls, want) {
|
||||||
|
t.Fatalf("adapter calls = %#v, want %#v", runner.Calls, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
persisted := loadAnalyzeRecoveryManifest(t, baseStore, cfg)
|
||||||
|
if persisted.Stages["analyze"].Status != manifest.StatusFailed {
|
||||||
|
t.Fatalf("aggregate analyze = %#v, want failed", persisted.Stages["analyze"])
|
||||||
|
}
|
||||||
|
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||||
|
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "encounters", manifest.AnalyzeArtifactFailed)
|
||||||
|
if _, exists := persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"]; exists {
|
||||||
|
t.Fatalf("unattempted quest log appeared: %#v", persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"])
|
||||||
|
}
|
||||||
|
for _, name := range []string{"publish", "notify"} {
|
||||||
|
if got := persisted.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type analyzeCheckpointRunner struct {
|
||||||
|
Calls []string
|
||||||
|
FailPrompt string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type failedAnalyzeSaveStore struct {
|
||||||
|
manifest.Store
|
||||||
|
Err error
|
||||||
|
Failed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type currentAnalyzeCheckpointFailingStore struct {
|
||||||
|
manifest.Store
|
||||||
|
CurrentLimit int
|
||||||
|
Err error
|
||||||
|
Failed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *currentAnalyzeCheckpointFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||||
|
if !s.Failed {
|
||||||
|
current := 0
|
||||||
|
if record := m.Stages["analyze"]; record != nil && record.Status == manifest.StatusRunning {
|
||||||
|
for _, artifact := range record.AnalyzeArtifacts {
|
||||||
|
if artifact.Status == manifest.AnalyzeArtifactCurrent {
|
||||||
|
current++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current >= s.CurrentLimit {
|
||||||
|
s.Failed = true
|
||||||
|
return s.Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s.Store.Save(ctx, path, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failedAnalyzeSaveStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||||
|
if record := m.Stages["analyze"]; !s.Failed && record != nil && record.Status == manifest.StatusFailed {
|
||||||
|
s.Failed = true
|
||||||
|
return s.Err
|
||||||
|
}
|
||||||
|
return s.Store.Save(ctx, path, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *analyzeCheckpointRunner) RunArtifact(
|
||||||
|
ctx context.Context,
|
||||||
|
req scriptorium.RunArtifactRequest,
|
||||||
|
) (scriptorium.ArtifactResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return scriptorium.ArtifactResult{}, err
|
||||||
|
}
|
||||||
|
r.Calls = append(r.Calls, req.PromptID)
|
||||||
|
if req.PromptID == r.FailPrompt {
|
||||||
|
return scriptorium.ArtifactResult{}, r.Err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(req.OutputPath), 0o755); err != nil {
|
||||||
|
return scriptorium.ArtifactResult{}, err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(req.OutputPath, []byte("generated "+req.PromptID+"\n"), 0o644); err != nil {
|
||||||
|
return scriptorium.ArtifactResult{}, err
|
||||||
|
}
|
||||||
|
return scriptorium.ArtifactResult{
|
||||||
|
OutputPath: req.OutputPath,
|
||||||
|
CommandMode: scriptorium.CommandModeRun,
|
||||||
|
PromptID: req.PromptID,
|
||||||
|
ProfileID: req.ProfileID,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*analyzeCheckpointRunner) RenderArtifact(
|
||||||
|
context.Context,
|
||||||
|
scriptorium.RenderArtifactRequest,
|
||||||
|
) (scriptorium.ArtifactResult, error) {
|
||||||
|
return scriptorium.ArtifactResult{}, errors.New("unexpected render")
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkpointRecoveryArtifacts(revision string) map[string]config.ScriptoriumArtifactConfig {
|
||||||
|
return map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"chronicle": {
|
||||||
|
Enabled: true, PromptID: "chronicle-" + revision, OutputPath: "artifacts/chronicle.md",
|
||||||
|
},
|
||||||
|
"encounters": {
|
||||||
|
Enabled: true, DependsOn: []string{"chronicle"}, PromptID: "encounters-" + revision,
|
||||||
|
OutputPath: "artifacts/encounters.md",
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"chronicle": {Source: "narratio.artifact.chronicle", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"quest_log": {
|
||||||
|
Enabled: true, DependsOn: []string{"encounters"}, PromptID: "quest-log-" + revision,
|
||||||
|
OutputPath: "artifacts/quest_log.md",
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"encounters": {Source: "narratio.artifact.encounters", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"player_handout": {
|
||||||
|
Enabled: true, DependsOn: []string{"quest_log"}, PromptID: "player-handout-" + revision,
|
||||||
|
OutputPath: "artifacts/player_handout.md",
|
||||||
|
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||||
|
"quest_log": {Source: "narratio.artifact.quest_log", Required: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"unrelated": {
|
||||||
|
Enabled: true, PromptID: "unrelated", OutputPath: "artifacts/unrelated.md",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustSelectStage(t *testing.T, name string) stage.Stage {
|
||||||
|
t.Helper()
|
||||||
|
selected, err := stage.Select(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Select(%q) error = %v", name, err)
|
||||||
|
}
|
||||||
|
return selected
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadAnalyzeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
m, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load session manifest: %v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertAnalyzeArtifactStatus(
|
||||||
|
t *testing.T,
|
||||||
|
record *manifest.StageRecord,
|
||||||
|
key string,
|
||||||
|
want manifest.AnalyzeArtifactStatus,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
got, exists := record.AnalyzeArtifacts[key]
|
||||||
|
if !exists || got.Status != want {
|
||||||
|
t.Fatalf("artifact %q = %#v, want status %q", key, got, want)
|
||||||
|
}
|
||||||
|
if want != manifest.AnalyzeArtifactCurrent && got.Output != nil {
|
||||||
|
t.Fatalf("artifact %q retained output authority: %#v", key, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStageRecords(m *manifest.Manifest, names ...string) map[string]manifest.StageRecord {
|
||||||
|
clones := make(map[string]manifest.StageRecord, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
clones[name] = *m.Stages[name]
|
||||||
|
}
|
||||||
|
return clones
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStageRecordsUnchanged(t *testing.T, m *manifest.Manifest, want map[string]manifest.StageRecord) {
|
||||||
|
t.Helper()
|
||||||
|
for name, expected := range want {
|
||||||
|
got := m.Stages[name]
|
||||||
|
if got == nil {
|
||||||
|
t.Fatalf("upstream stage %q is missing", name)
|
||||||
|
}
|
||||||
|
if diff := cmp.Diff(expected, *got); diff != "" {
|
||||||
|
t.Fatalf("upstream stage %q changed (-want +got):\n%s", name, diff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
|
||||||
)
|
|
||||||
|
|
||||||
type validatedAnalyzeProjection struct {
|
|
||||||
session map[string]manifest.AnalyzeArtifactRecord
|
|
||||||
invocation map[string]manifest.AnalyzeArtifactRecord
|
|
||||||
}
|
|
||||||
|
|
||||||
type analyzeStateSnapshot struct {
|
|
||||||
version int
|
|
||||||
records map[string]manifest.AnalyzeArtifactRecord
|
|
||||||
}
|
|
||||||
|
|
||||||
func captureAnalyzeState(manifestValue *manifest.Manifest, stageName string) analyzeStateSnapshot {
|
|
||||||
if manifestValue == nil || stageName != "analyze" || manifestValue.Stages["analyze"] == nil {
|
|
||||||
return analyzeStateSnapshot{}
|
|
||||||
}
|
|
||||||
record := manifestValue.Stages["analyze"]
|
|
||||||
return analyzeStateSnapshot{
|
|
||||||
version: record.AnalyzeStateVersion,
|
|
||||||
records: manifest.CloneAnalyzeArtifactCollection(record.AnalyzeArtifacts),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func restoreAnalyzeState(manifestValue *manifest.Manifest, snapshot analyzeStateSnapshot) {
|
|
||||||
if manifestValue == nil || manifestValue.Stages["analyze"] == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
record := manifestValue.Stages["analyze"]
|
|
||||||
record.AnalyzeStateVersion = snapshot.version
|
|
||||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(snapshot.records)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateSuccessfulAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
|
||||||
if result == nil || result.AnalyzeState == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if stageName != "analyze" {
|
|
||||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection", stageName)
|
|
||||||
}
|
|
||||||
if result.Disposition == stage.StageDispositionSkipped {
|
|
||||||
return nil, fmt.Errorf("skipped analyze result cannot contain analyze-owned state projection")
|
|
||||||
}
|
|
||||||
if len(result.Outputs) != 0 {
|
|
||||||
return nil, fmt.Errorf("analyze result with state projection cannot contain ordinary outputs")
|
|
||||||
}
|
|
||||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateFailedAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
|
||||||
if result == nil || result.AnalyzeState == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if stageName != "analyze" {
|
|
||||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection with an error", stageName)
|
|
||||||
}
|
|
||||||
if result.Disposition != stage.StageDispositionSucceeded || result.SkipReason != "" || len(result.Outputs) != 0 || len(result.Logs) != 0 || len(result.GeneratedConfigs) != 0 || len(result.Metadata) != 0 {
|
|
||||||
return nil, fmt.Errorf("analyze result with an error may contain only analyze-owned state projection")
|
|
||||||
}
|
|
||||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateAndCloneAnalyzeProjection(projection *stage.AnalyzeStateProjection) (*validatedAnalyzeProjection, error) {
|
|
||||||
session := manifest.CloneAnalyzeArtifactCollection(projection.Session)
|
|
||||||
invocation := manifest.CloneAnalyzeArtifactCollection(projection.Invocation)
|
|
||||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, session); err != nil {
|
|
||||||
return nil, fmt.Errorf("validate reconciled session analyze state: %w", err)
|
|
||||||
}
|
|
||||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, invocation); err != nil {
|
|
||||||
return nil, fmt.Errorf("validate invocation analyze state: %w", err)
|
|
||||||
}
|
|
||||||
for key, invocationRecord := range invocation {
|
|
||||||
sessionRecord, ok := session[key]
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("invocation analyze artifact %q is absent from reconciled session state", key)
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(invocationRecord, sessionRecord) {
|
|
||||||
return nil, fmt.Errorf("invocation analyze artifact %q contradicts reconciled session state", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &validatedAnalyzeProjection{session: session, invocation: invocation}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func applyAnalyzeProjection(
|
|
||||||
sessionManifest *manifest.Manifest,
|
|
||||||
runManifest *manifest.RunManifest,
|
|
||||||
projection *validatedAnalyzeProjection,
|
|
||||||
) {
|
|
||||||
if projection == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if sessionManifest != nil && sessionManifest.Stages["analyze"] != nil {
|
|
||||||
record := sessionManifest.Stages["analyze"]
|
|
||||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
|
||||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.session)
|
|
||||||
}
|
|
||||||
if runManifest != nil && runManifest.Stages["analyze"] != nil {
|
|
||||||
record := runManifest.Stages["analyze"]
|
|
||||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
|
||||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.invocation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func analyzeProjectionOutputs(records map[string]manifest.AnalyzeArtifactRecord, producerRunID string) []manifest.ArtifactRecord {
|
|
||||||
keys := make([]string, 0, len(records))
|
|
||||||
for key, record := range records {
|
|
||||||
if record.Status != manifest.AnalyzeArtifactCurrent || record.Output == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if producerRunID != "" && record.ProducerRunID != producerRunID {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
keys = append(keys, key)
|
|
||||||
}
|
|
||||||
sort.Strings(keys)
|
|
||||||
outputs := make([]manifest.ArtifactRecord, 0, len(keys))
|
|
||||||
for _, key := range keys {
|
|
||||||
record := manifest.CloneAnalyzeArtifactCollection(map[string]manifest.AnalyzeArtifactRecord{key: records[key]})[key]
|
|
||||||
output := *record.Output
|
|
||||||
if output.ProducerRunID == "" {
|
|
||||||
output.ProducerRunID = record.ProducerRunID
|
|
||||||
}
|
|
||||||
outputs = append(outputs, output)
|
|
||||||
}
|
|
||||||
return outputs
|
|
||||||
}
|
|
||||||
@@ -1,346 +0,0 @@
|
|||||||
package app
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
|
||||||
)
|
|
||||||
|
|
||||||
type projectionStage struct {
|
|
||||||
name string
|
|
||||||
run func(*stage.Env, *manifest.Manifest) (*stage.StageResult, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s projectionStage) Name() string { return s.name }
|
|
||||||
func (s projectionStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
return s.run(env, m)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesProjectsSeparateSessionAndInvocationAnalyzeState(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
oldAt := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
|
||||||
oldRecord := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", oldAt)
|
|
||||||
|
|
||||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
staleRecord := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactStale, m.RunID, time.Now().UTC())
|
|
||||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"player_handout": oldRecord,
|
|
||||||
"quest_log": staleRecord,
|
|
||||||
"session_recap": newRecord,
|
|
||||||
}
|
|
||||||
return &stage.StageResult{
|
|
||||||
Logs: []string{"aggregate-analyze.log"},
|
|
||||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: session,
|
|
||||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"player_handout": oldRecord,
|
|
||||||
"session_recap": newRecord,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}}
|
|
||||||
|
|
||||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
store := &manifest.LocalStore{}
|
|
||||||
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Load(session) error = %v", err)
|
|
||||||
}
|
|
||||||
analyze := sessionManifest.Stages["analyze"]
|
|
||||||
if analyze.AnalyzeStateVersion != manifest.AnalyzeStateContractVersion || len(analyze.AnalyzeArtifacts) != 3 {
|
|
||||||
t.Fatalf("session analyze state = %#v", analyze)
|
|
||||||
}
|
|
||||||
if got := analyzeArtifactOutputKeys(analyze.Outputs); !reflect.DeepEqual(got, []string{"player_handout", "session_recap"}) {
|
|
||||||
t.Fatalf("session aggregate outputs = %#v, want current records only", got)
|
|
||||||
}
|
|
||||||
if len(analyze.Logs) != 1 || analyze.Logs[0] != "aggregate-analyze.log" {
|
|
||||||
t.Fatalf("session aggregate logs = %#v", analyze.Logs)
|
|
||||||
}
|
|
||||||
|
|
||||||
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadRun() error = %v", err)
|
|
||||||
}
|
|
||||||
runAnalyze := runManifest.Stages["analyze"]
|
|
||||||
if len(runAnalyze.AnalyzeArtifacts) != 2 || runAnalyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent {
|
|
||||||
t.Fatalf("invocation analyze state = %#v", runAnalyze.AnalyzeArtifacts)
|
|
||||||
}
|
|
||||||
if got := analyzeArtifactOutputKeys(runAnalyze.Outputs); !reflect.DeepEqual(got, []string{"session_recap"}) {
|
|
||||||
t.Fatalf("invocation outputs = %#v, want produced artifact only", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesPersistsRestrictedAnalyzeStateOnPartialError(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
|
||||||
seed := manifest.New(cfg.Session.SessionID, now)
|
|
||||||
seed.Campaign = cfg.Session.Campaign
|
|
||||||
seed.MarkStageSucceeded("analyze", now, nil)
|
|
||||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
|
||||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"player_handout": appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now),
|
|
||||||
}
|
|
||||||
seed.MarkStageSucceeded("publish", now, nil)
|
|
||||||
saveBoundedManifest(t, cfg, seed)
|
|
||||||
|
|
||||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
unrelated := seed.Stages["analyze"].AnalyzeArtifacts["player_handout"]
|
|
||||||
completed := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
failed := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactFailed, m.RunID, time.Now().UTC())
|
|
||||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"player_handout": unrelated,
|
|
||||||
"quest_log": failed,
|
|
||||||
"session_recap": completed,
|
|
||||||
}
|
|
||||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: session,
|
|
||||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"quest_log": failed,
|
|
||||||
"session_recap": completed,
|
|
||||||
},
|
|
||||||
}}, errors.New("quest log failed")
|
|
||||||
}}
|
|
||||||
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "quest log failed") {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
store := &manifest.LocalStore{}
|
|
||||||
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Load(session) error = %v", err)
|
|
||||||
}
|
|
||||||
analyze := loaded.Stages["analyze"]
|
|
||||||
if analyze.Status != manifest.StatusFailed || len(analyze.Outputs) != 0 {
|
|
||||||
t.Fatalf("aggregate analyze state = %#v, want failed without outputs", analyze)
|
|
||||||
}
|
|
||||||
if analyze.AnalyzeArtifacts["player_handout"].Status != manifest.AnalyzeArtifactCurrent ||
|
|
||||||
analyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent ||
|
|
||||||
analyze.AnalyzeArtifacts["quest_log"].Status != manifest.AnalyzeArtifactFailed {
|
|
||||||
t.Fatalf("partial session projection = %#v", analyze.AnalyzeArtifacts)
|
|
||||||
}
|
|
||||||
if loaded.Stages["publish"].Status != manifest.StatusStale {
|
|
||||||
t.Fatalf("publish status = %q, want stale", loaded.Stages["publish"].Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
runsDir := artifacts.SessionRunsDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
|
||||||
entries, err := os.ReadDir(runsDir)
|
|
||||||
if err != nil || len(entries) != 1 {
|
|
||||||
t.Fatalf("run directory entries = %#v, error = %v", entries, err)
|
|
||||||
}
|
|
||||||
runManifest, err := store.LoadRun(context.Background(), filepath.Join(runsDir, entries[0].Name(), "manifest.json"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadRun() error = %v", err)
|
|
||||||
}
|
|
||||||
runAnalyze := runManifest.Stages["analyze"]
|
|
||||||
if runAnalyze.Status != manifest.StatusFailed || len(runAnalyze.AnalyzeArtifacts) != 2 || len(runAnalyze.Outputs) != 0 {
|
|
||||||
t.Fatalf("partial invocation projection = %#v", runAnalyze)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesRejectsInvalidAnalyzeProjectionWithoutReplacingPriorState(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
|
||||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
|
||||||
seed := manifest.New(cfg.Session.SessionID, now)
|
|
||||||
seed.Campaign = cfg.Session.Campaign
|
|
||||||
seed.MarkStageSucceeded("analyze", now, nil)
|
|
||||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
|
||||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
|
||||||
saveBoundedManifest(t, cfg, seed)
|
|
||||||
|
|
||||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
invalid := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
invalid.Output.Checksum = "invalid"
|
|
||||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
|
||||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
|
||||||
}}, errors.New("analysis failed")
|
|
||||||
}}
|
|
||||||
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "checksum") {
|
|
||||||
t.Fatalf("executeStages() error = %v, want projection validation failure", err)
|
|
||||||
}
|
|
||||||
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
|
||||||
if loadErr != nil {
|
|
||||||
t.Fatalf("Load() error = %v", loadErr)
|
|
||||||
}
|
|
||||||
if len(loaded.Stages["analyze"].AnalyzeArtifacts) != 1 || !reflect.DeepEqual(loaded.Stages["analyze"].AnalyzeArtifacts["player_handout"], prior) {
|
|
||||||
t.Fatalf("prior state replaced by invalid projection: %#v", loaded.Stages["analyze"].AnalyzeArtifacts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesRollsBackAnalyzeAuthorityWhenProjectionSaveFails(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
|
||||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
|
||||||
seed := manifest.New(cfg.Session.SessionID, now)
|
|
||||||
seed.Campaign = cfg.Session.Campaign
|
|
||||||
seed.MarkStageSucceeded("analyze", now, nil)
|
|
||||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
|
||||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
|
||||||
saveBoundedManifest(t, cfg, seed)
|
|
||||||
|
|
||||||
stageReturned := false
|
|
||||||
store := &analyzeProjectionFailingStore{delegate: &manifest.LocalStore{}, shouldFail: func(m *manifest.Manifest) bool {
|
|
||||||
return stageReturned && m.Stages["analyze"] != nil && m.Stages["analyze"].Status == manifest.StatusSucceeded && m.Stages["analyze"].AnalyzeArtifacts["session_recap"].Status == manifest.AnalyzeArtifactCurrent
|
|
||||||
}}
|
|
||||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
stageReturned = true
|
|
||||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: map[string]manifest.AnalyzeArtifactRecord{
|
|
||||||
"player_handout": prior,
|
|
||||||
"session_recap": newRecord,
|
|
||||||
},
|
|
||||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": newRecord},
|
|
||||||
}}, nil
|
|
||||||
}}
|
|
||||||
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{
|
|
||||||
Force: true,
|
|
||||||
Env: &Env{ManifestStore: store},
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "injected analyze projection save failure") {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
if !store.failed {
|
|
||||||
t.Fatal("projection persistence failure was not injected")
|
|
||||||
}
|
|
||||||
loaded, loadErr := store.delegate.Load(context.Background(), manifestPathFor(cfg))
|
|
||||||
if loadErr != nil {
|
|
||||||
t.Fatalf("Load() error = %v", loadErr)
|
|
||||||
}
|
|
||||||
analyze := loaded.Stages["analyze"]
|
|
||||||
if analyze.Status != manifest.StatusFailed || len(analyze.AnalyzeArtifacts) != 1 || !reflect.DeepEqual(analyze.AnalyzeArtifacts["player_handout"], prior) {
|
|
||||||
t.Fatalf("durable analyze state after rollback = %#v", analyze)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesRejectsAnalyzeProjectionFromOtherStage(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
stageToRun := projectionStage{name: "prepare", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
|
||||||
}}, nil
|
|
||||||
}}
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "returned analyze-owned state projection") {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesRejectsContradictoryAnalyzeResultWithError(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
|
||||||
return &stage.StageResult{
|
|
||||||
Outputs: []artifacts.Ref{{Kind: "session_recap", RelativePath: "artifacts/session-recap.md"}},
|
|
||||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
|
||||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
|
||||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
|
||||||
},
|
|
||||||
}, errors.New("analysis failed")
|
|
||||||
}}
|
|
||||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "may contain only analyze-owned state projection") {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteStagesExposesSelectedForceDecisionToStage(t *testing.T) {
|
|
||||||
for _, force := range []bool{false, true} {
|
|
||||||
t.Run(strings.ToLower(strings.TrimSpace(map[bool]string{false: "ordinary", true: "forced"}[force])), func(t *testing.T) {
|
|
||||||
cfg := testConfig(t)
|
|
||||||
captured := !force
|
|
||||||
stageToRun := projectionStage{name: "prepare", run: func(env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
|
||||||
captured = env.Force
|
|
||||||
return &stage.StageResult{}, nil
|
|
||||||
}}
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: force}); err != nil {
|
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
|
||||||
}
|
|
||||||
if captured != force {
|
|
||||||
t.Fatalf("stage env force = %v, want %v", captured, force)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func appAnalyzeRecord(key string, status manifest.AnalyzeArtifactStatus, producerRunID string, at time.Time) manifest.AnalyzeArtifactRecord {
|
|
||||||
record := manifest.AnalyzeArtifactRecord{
|
|
||||||
Key: key,
|
|
||||||
Status: status,
|
|
||||||
ProducerRunID: producerRunID,
|
|
||||||
UpdatedAt: at,
|
|
||||||
}
|
|
||||||
if status == manifest.AnalyzeArtifactFailed {
|
|
||||||
record.Error = "scriptorium failed"
|
|
||||||
return record
|
|
||||||
}
|
|
||||||
if status != manifest.AnalyzeArtifactCurrent {
|
|
||||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
|
||||||
record.Fingerprint = strings.Repeat("b", 64)
|
|
||||||
return record
|
|
||||||
}
|
|
||||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
|
||||||
record.Fingerprint = strings.Repeat("a", 64)
|
|
||||||
record.OutputSize = 42
|
|
||||||
record.Output = &manifest.ArtifactRecord{
|
|
||||||
Kind: key,
|
|
||||||
SourceID: artifacts.ConfiguredArtifactSourceID(key),
|
|
||||||
LocalPath: "artifacts/" + strings.ReplaceAll(key, "_", "-") + ".md",
|
|
||||||
ProducerRunID: producerRunID,
|
|
||||||
Checksum: strings.Repeat("c", 64),
|
|
||||||
Contract: &artifactmodel.ContractMetadata{
|
|
||||||
MediaType: "text/markdown", SchemaID: "narratio." + key, SchemaVersion: "1",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return record
|
|
||||||
}
|
|
||||||
|
|
||||||
func analyzeArtifactOutputKeys(outputs []manifest.ArtifactRecord) []string {
|
|
||||||
keys := make([]string, 0, len(outputs))
|
|
||||||
for _, output := range outputs {
|
|
||||||
keys = append(keys, strings.TrimPrefix(output.SourceID, "narratio.artifact."))
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
type analyzeProjectionFailingStore struct {
|
|
||||||
delegate *manifest.LocalStore
|
|
||||||
shouldFail func(*manifest.Manifest) bool
|
|
||||||
failed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *analyzeProjectionFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
|
||||||
return s.delegate.Create(ctx, sessionID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *analyzeProjectionFailingStore) Load(ctx context.Context, path string) (*manifest.Manifest, error) {
|
|
||||||
return s.delegate.Load(ctx, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *analyzeProjectionFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
|
||||||
if !s.failed && s.shouldFail != nil && s.shouldFail(m) {
|
|
||||||
s.failed = true
|
|
||||||
return errors.New("injected analyze projection save failure")
|
|
||||||
}
|
|
||||||
return s.delegate.Save(ctx, path, m)
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user